#include "DocumentTracker.h" const char* DocumentTracker::camerasDirOffset = "devices\\cameras\\"; string DocumentTracker::RootDirectory = ""; static int index = 0; static const double pi = 3.14159265358979323846; static FILE* f; static int iter = 0; inline static double square(int a) { return a * a; } DocumentTracker::DocumentTracker(){ } DocumentTracker::~DocumentTracker(){ } bool DocumentTracker::init(){ cameraID = 6090168; eventBufIndex = -1; //count = 0; firstTime = true; pointIndex = 0; currentDocument = -1; // a hack, this value will only go up to 3 firstNumDetectedFeatures = 0; //initialize all the open cv images for(int i = 0; i < BUFFERS; i ++){ eventBuf[i] = cvCreateImage(cvSize(CAMWIDTH, CAMHEIGHT), IPL_DEPTH_8U, 1); eventBufAbsDiff[i] = 0; } cvPredictionIm = cvCreateImage(cvSize(CAMWIDTH, CAMHEIGHT), IPL_DEPTH_8U, 1); tempImage1 = cvCreateImage(cvSize(CAMWIDTH, CAMHEIGHT), IPL_DEPTH_8U, 3); tempImage2 = cvCreateImage(cvSize(CAMWIDTH, CAMHEIGHT), IPL_DEPTH_8U, 3); eigenImageFeatureDet = cvCreateImage(cvSize(CAMWIDTH, CAMHEIGHT), IPL_DEPTH_32F, 1); tempImageFeatureDet = cvCreateImage(cvSize(CAMWIDTH, CAMHEIGHT), IPL_DEPTH_32F, 1); cvPyr1 = cvCreateImage(cvSize(CAMWIDTH + 8, CAMHEIGHT / 3), IPL_DEPTH_8U, 1); cvPyr2 = cvCreateImage(cvSize(CAMWIDTH + 8, CAMHEIGHT / 3), IPL_DEPTH_8U, 1); currFeatures = new CvPoint2D32f[MAX_FEATURES_TO_TRACK]; prevFeatures = new CvPoint2D32f[MAX_FEATURES_TO_TRACK]; firstFeatures = new CvPoint2D32f[MAX_FEATURES_TO_TRACK]; status = new char[MAX_FEATURES_TO_TRACK]; error = new float[MAX_FEATURES_TO_TRACK]; RootDirectory = WAVUtil::getWAVDirectory(); f = fopen("events.txt","w"); for(int i = 0; i < MAX_NUM_DOC + 2; i++){ for(int j = 0; j < MAX_NUM_DOC; j++){ if(i < MAX_NUM_DOC) dag[i][j] = 0; else dag[i][j] = -1; } } if(RootDirectory.length() > 0){ CamUtil::setCamDir(RootDirectory + camerasDirOffset); return true; } else return false; } bool DocumentTracker::start(){ return cb.start(cameraID); } void DocumentTracker::stop(){ cb.stop(); clear(); fclose(f); } int DocumentTracker::writeOpticalFlow(IplImage* frame1, IplImage* frame2){ numDetectedFeatures = MAX_FEATURES_TO_TRACK; cvConvertImage(frame2, tempImage2); if(firstTime == true){ cvGoodFeaturesToTrack(frame1, eigenImageFeatureDet, tempImageFeatureDet, firstFeatures, &numDetectedFeatures, 0.005, 3); firstTime = false; firstNumDetectedFeatures = numDetectedFeatures; for(int i = 0; i < numDetectedFeatures; i++){ prevFeatures[i].x = firstFeatures[i].x; prevFeatures[i].y = firstFeatures[i].y; } } else{ for(int i = 0; i < numDetectedFeatures; i++){ prevFeatures[i].x = currFeatures[i].x; prevFeatures[i].y = currFeatures[i].y; } } //calculate the movement int flags = CV_LKFLOW_INITIAL_GUESSES; cvCalcOpticalFlowPyrLK(frame1, frame2, cvPyr1, cvPyr2, prevFeatures, currFeatures, numDetectedFeatures, cvSize(3, 3), 5, status, error, cvTermCriteria(CV_TERMCRIT_ITER|CV_TERMCRIT_EPS, 20, 0.3f), 0); numFeaturesFound = 0; for(int i = 0; i < numDetectedFeatures; i++) { /* If Pyramidal Lucas Kanade didn't really find the feature, skip it. */ if ( status[i] == 0 ) continue; CvPoint p,q; p.x = (int) prevFeatures[i].x; p.y = (int) prevFeatures[i].y; q.x = (int) currFeatures[i].x; q.y = (int) currFeatures[i].y; double angle; angle = atan2( (double) p.y - q.y, (double) p.x - q.x ); double hypotenuse; hypotenuse = sqrt( square(p.y - q.y) + square(p.x - q.x) ); //dont include any points that have a hypotenuse less than 2 (false positives) if(hypotenuse < 2) continue; //cluster point; //point.qx = q.x; //point.qy = q.y; //point.px = p.x; //point.py = p.y; //point.hyp = hypotenuse; //point.ang = point.ang; //drawLineOnImage(tempImage2,point); numFeaturesFound++; } //char buf[200]; //sprintf_s(buf,200,"opticalflow%d.bmp",index++); //cvSaveImage(buf,tempImage2); return numFeaturesFound; } void DocumentTracker::computePoints(CvPoint2D32f* ppoints, CvPoint2D32f* qpoints){ CvPoint p,q; for(int i = 0; i < firstNumDetectedFeatures; i++){ p.x = (int) ppoints[i].x; p.y = (int) ppoints[i].y; q.x = (int) qpoints[i].x; q.y = (int) qpoints[i].y; //fprintf(f,"numDetectedFeatures = %d\n",numDetectedFeatures); //fprintf(f,"currFeatures[%d].x = %f\n",i,qpoints[i].x); //fprintf(f,"currFeatures[%d].y = %f\n",i,qpoints[i].y); //fprintf(f,"firstFeatures[%d].x = %f\n",i,ppoints[i].x); //fprintf(f,"firstFeatures[%d].y = %f\n",i,ppoints[i].y); //fprintf(f,"q.x = %d\n",q.x); //fprintf(f,"q.y = %d\n",q.y); //fprintf(f,"p.x = %d\n",p.x); //fprintf(f,"p.y = %d\n",p.y); double angle; angle = atan2( (double) p.y - q.y, (double) p.x - q.x ); double hypotenuse; hypotenuse = sqrt( square(p.y - q.y) + square(p.x - q.x) ); //fprintf(f,"angle = %f\n",angle); //fprintf(f,"hypotenuse = %f\n",hypotenuse); if(hypotenuse < 2) continue; points[pointIndex].hyp = hypotenuse; points[pointIndex].ang = angle*100; //i do *100 to make the K Means work better points[pointIndex].px = p.x; points[pointIndex].py = p.y; points[pointIndex].qx = q.x; points[pointIndex].qy = q.y; points[pointIndex++].id = -1; } cout << "point index = " << pointIndex << endl; } void DocumentTracker::computeAvgPoints(){ int sumX = 0; int sumY = 0; for(int i = 0; i < pointIndex; i++){ sumX += points[i].qx; sumY += points[i].qy; } if(pointIndex > 0){ sumX /= pointIndex; sumY /= pointIndex; dag[MAX_NUM_DOC][currentDocument] = sumX; dag[MAX_NUM_DOC+1][currentDocument] = sumY; } } void DocumentTracker::printDAG(){ cout << "DAG:" << endl; for(int i = 0; i < MAX_NUM_DOC + 2; i++){ cout << "["; for(int j = 0; j < MAX_NUM_DOC; j++){ if(j != MAX_NUM_DOC - 1) cout << dag[i][j] << "] ["; else cout << dag[i][j]; } cout << "]" << endl; } } void DocumentTracker::recomputeDAG(int eventType){ int currentDocX = dag[MAX_NUM_DOC][currentDocument]; int currentDocY = dag[MAX_NUM_DOC+1][currentDocument]; if(eventType == 2){ for(int i = 0; i < MAX_NUM_DOC; i++){ dag[currentDocument][i] = 0; } } else if(eventType == 1){ //do nothing } else{ //want to find any documents that are "close" to the current one //if so they are overlapping for(int i = 0; i < MAX_NUM_DOC; i++){ int tempX = dag[MAX_NUM_DOC][i]; int tempY = dag[MAX_NUM_DOC+1][i]; //compute euclidean distance int tempD = sqrt( square(tempY - currentDocY) + square(tempX - currentDocX) ); if(tempD < 30) dag[currentDocument][i] = 1; else dag[currentDocument][i] = 0; } } } void DocumentTracker::drawLinesOnImage(IplImage* image, CvPoint2D32f* ppoints, CvPoint2D32f* qpoints){ cout << "draw lines on image" << endl; for(int i = 0; i < numDetectedFeatures; i++){ int line_thickness; line_thickness = 1; CvScalar line_color; line_color = CV_RGB(255,0,0); CvPoint p,q; p.x = (int) ppoints[i].x; p.y = (int) ppoints[i].y; q.x = (int) qpoints[i].x; q.y = (int) qpoints[i].y; double angle; angle = atan2( (double) p.y - q.y, (double) p.x - q.x ); double hypotenuse; hypotenuse = sqrt( square(p.y - q.y) + square(p.x - q.x) ); if(hypotenuse < 2) continue; ///* Here we lengthen the arrow by a factor of three. */ //q.x = (int) (p.x - 3 * hypotenuse * cos(angle)); //q.y = (int) (p.y - 3 * hypotenuse * sin(angle)); ///* Now we draw the main line of the arrow. */ cvLine(image, p, q, line_color, line_thickness, CV_AA, 0 ); p.x = (int) (q.x + 9 * cos(angle + pi / 4)); p.y = (int) (q.y + 9 * sin(angle + pi / 4)); cvLine(image, p, q, line_color, line_thickness, CV_AA, 0 ); p.x = (int) (q.x + 9 * cos(angle - pi / 4)); p.y = (int) (q.y + 9 * sin(angle - pi / 4)); cvLine(image, p, q, line_color, line_thickness, CV_AA, 0 ); } } void DocumentTracker::drawLineOnImage(IplImage* image, cluster point){ int line_thickness; line_thickness = 1; CvScalar line_color; line_color = CV_RGB(255,0,0); CvPoint p,q; p.x = point.px; p.y = point.py; q.x = point.qx; q.y = point.qy; double angle; angle = atan2( (double) p.y - q.y, (double) p.x - q.x ); double hypotenuse; hypotenuse = sqrt( square(p.y - q.y) + square(p.x - q.x) ); ///* Here we lengthen the arrow by a factor of three. */ q.x = (int) (p.x - 3 * hypotenuse * cos(angle)); q.y = (int) (p.y - 3 * hypotenuse * sin(angle)); cvLine(image, p, q, line_color, line_thickness, CV_AA, 0 ); p.x = (int) (q.x + 9 * cos(angle + pi / 4)); p.y = (int) (q.y + 9 * sin(angle + pi / 4)); cvLine(image, p, q, line_color, line_thickness, CV_AA, 0 ); p.x = (int) (q.x + 9 * cos(angle - pi / 4)); p.y = (int) (q.y + 9 * sin(angle - pi / 4)); cvLine(image, p, q, line_color, line_thickness, CV_AA, 0 ); } int DocumentTracker::classifyevent(){ boolean moveOccurred = false; int retVal = -1; CvFont font; cvInitFont( &font, CV_FONT_VECTOR0, 1.0f,1.0f, 0, 2 ); CvScalar line_color; line_color = CV_RGB(255,0,0); char buf[200]; char textbuf[200]; currentDocument = (++currentDocument % MAX_NUM_DOC); cvConvertImage(eventBuf[eventBufIndex - 1], tempImage1); /*sprintf_s(buf,200,"last.bmp"); cvSaveImage(buf,tempImage1);*/ computePoints(firstFeatures,currFeatures); int largestIndex = kmeansCluster(); int numPoints = 0; cvConvertImage(eventBuf[eventBufIndex - 1], tempImage2); for(int i = 0; i < pointIndex; i++) { cvCircle(tempImage2, cvPoint(cvRound(firstFeatures[i].x), cvRound(firstFeatures[i].y)), 2, CV_RGB(0, 0, 255)); cvCircle(tempImage2, cvPoint(cvRound(currFeatures[i].x), cvRound(currFeatures[i].y)), 2, CV_RGB(255, 0, 0)); } for(int i = 0; i < pointIndex; i++){ if(points[i].id == largestIndex){ points[i].ang /= 100; /*drawLineOnImage(tempImage1,points[i]); drawLineOnImage(tempImage2,points[i]);*/ numPoints++; //fprintf(f,"numPoints = %d\n",numPoints); //fprintf(f,"points[%d].hyp = %f\n",i,points[i].hyp); //fprintf(f,"points[%d].ang = %f\n",i,points[i].ang); //fprintf(f,"points[%d].qx = %f\n",i,points[i].qx); //fprintf(f,"points[%d].qy = %f\n",i,points[i].qy); //fprintf(f,"points[%d].px = %f\n",i,points[i].px); //fprintf(f,"points[%d].py = %f\n",i,points[i].py); } } //sprintf_s(buf,200,"first.bmp"); //cvSaveImage(buf,tempImage2); cout << "NumPoints = " << numPoints << endl; if(numPoints > 100) moveOccurred = true; if(moveOccurred){ cout << "Document " << currentDocument << " has moved " << numPoints << endl;; /*for(int i = 0; i < pointIndex; i++) { if(points[i].id == largestIndex){ cvCircle(tempImage1, cvPoint(cvRound(points[i].px), cvRound(points[i].py)), 2, CV_RGB(0, 0, 255)); } else{ cvCircle(tempImage1, cvPoint(cvRound(currFeatures[i].x), cvRound(currFeatures[i].y)), 2, CV_RGB(0, 255, 0)); } }*/ //cvConvertImage(eventBuf[eventBufIndex - 1], tempImage1); sprintf_s(textbuf,200,"Document %d has moved",currentDocument); cvPutText(tempImage1,textbuf,cvPoint(200,300),&font,line_color); retVal = 0; } else { bool entryOccurred = true; int max = 0; int id = -1; //find the max of eventAbsDiff. if it is near the beginning it is an entry, if not it is an exit for(int i = 0; i < eventBufIndex; i++){ if(eventBufAbsDiff[i] > max){ max = eventBufAbsDiff[i]; id = i; } } int median = ((double)eventBufIndex/2) / eventBufIndex/2 < 0.5 ? floor((double)eventBufIndex/2) : ceil((double)eventBufIndex/2); if(id > median) entryOccurred = false; if(entryOccurred){ cout << "Document " << currentDocument << " has entered" << numPoints << " " << id << " " << median << " " << eventBufIndex << endl; /*for(int i = 0; i < pointIndex; i++) { if(points[i].id == largestIndex){ cvCircle(tempImage1, cvPoint(cvRound(points[i].qx), cvRound(points[i].qy)), 2, CV_RGB(0, 0, 255)); } else{ cvCircle(tempImage1, cvPoint(cvRound(currFeatures[i].x), cvRound(currFeatures[i].y)), 2, CV_RGB(0, 255, 0)); } }*/ //cvConvertImage(eventBuf[eventBufIndex - 1], tempImage1); sprintf_s(textbuf,200,"Document %d has entered",currentDocument); cvPutText(tempImage1,textbuf,cvPoint(200,300),&font,line_color); retVal = 1; } else{ cout << "Document " << currentDocument << " has exited" << numPoints << " " << id << " " << median << " " << eventBufIndex << endl; /*for(int i = 0; i < pointIndex; i++) { if(points[i].id == largestIndex){ cvCircle(tempImage1, cvPoint(cvRound(points[i].px), cvRound(points[i].py)), 2, CV_RGB(0, 0, 255)); } else{ cvCircle(tempImage1, cvPoint(cvRound(currFeatures[i].x), cvRound(currFeatures[i].y)), 2, CV_RGB(0, 255, 0)); } }*/ //cvConvertImage(eventBuf[eventBufIndex - 1], tempImage1); sprintf_s(textbuf,200,"Document %d has exited",currentDocument); cvPutText(tempImage1,textbuf,cvPoint(200,300),&font,line_color); retVal = 2; } } for(int i = 0; i < eventBufIndex - 1; i++){ sprintf_s(buf,200,"event%d.bmp",index++); cvSaveImage(buf,eventBuf[i]); } //used to add one second worth of video with the event type for( int i = 0; i < 15; i++){ sprintf_s(buf,200,"event%d.bmp",index++); cvSaveImage(buf,tempImage1); } //printDAG(); //computeAvgPoints(); //recomputeDAG(retVal); //printDAG(); return retVal; } void DocumentTracker::clear(){ for(int i = 0; i < BUFFERS; i ++){ cvReleaseImage(&eventBuf[i]); } for(int i = 0; i < BUFFERS; i ++){ eventBuf[i] = cvCreateImage(cvSize(CAMWIDTH, CAMHEIGHT), IPL_DEPTH_8U, 1); eventBufAbsDiff[i] = 0; } eventBufIndex = -1; firstTime = true; pointIndex = 0; } void DocumentTracker::run(){ int testIndex = 0; char buf[200]; int framesSinceEvent = 0; boolean eventOcc = false; uchar* data; while(1){ if(kbhit()){ char charpressed = getch(); if(charpressed == 'q' || charpressed == 27){ cout << "Exiting..." << endl; break; } } if(eventBufIndex < 0){ cb.getLatestImage(prevImage); cvPrevImage = cvCreateImageHeader(cvSize(CAMWIDTH, CAMHEIGHT), IPL_DEPTH_8U, 1); cvSetData(cvPrevImage, prevImage.image.image.pData, 1*CAMWIDTH); } else{ cvPrevImage = cvCreateImage(cvSize(CAMWIDTH, CAMHEIGHT), IPL_DEPTH_8U, 1); cvCopyImage(eventBuf[eventBufIndex],cvPrevImage); } cb.getLatestImage(currentImage); cvCurrentImage = cvCreateImageHeader(cvSize(CAMWIDTH, CAMHEIGHT), IPL_DEPTH_8U, 1); cvSetData(cvCurrentImage, currentImage.image.image.pData, 1*CAMWIDTH); cvFrameDiff = cvCreateImage(cvSize(CAMWIDTH, CAMHEIGHT), IPL_DEPTH_8U, 1); cvAbsDiff(cvCurrentImage,cvPrevImage,cvFrameDiff); //sprintf_s(buf,200,"framediff%d.bmp",index++); //cvSaveImage(buf,cvFrameDiff); data = (uchar *)cvFrameDiff->imageData; int nonZeroCount = 0; //note some of the values were above 0 so cvCountNonZero didnt work. //needed a higher threshold than 0 to do event detection for(int j = 0; j < cvFrameDiff->width*cvFrameDiff->height; j++){ if(data[j] > 30) nonZeroCount++; } //find the event as it occurs if(nonZeroCount > 4){ eventOcc = true; framesSinceEvent = 0; eventBufIndex++; if((eventBufIndex + 1) < BUFFERS){ if(eventBufIndex == 0){ cvCopyImage(cvPrevImage,eventBuf[eventBufIndex++]); } cvCopyImage(cvCurrentImage,eventBuf[eventBufIndex]); eventBufAbsDiff[eventBufIndex]= nonZeroCount; } numFeatures = writeOpticalFlow(eventBuf[eventBufIndex - 1],eventBuf[eventBufIndex]); } else { // an event has occurred before and so we need to classify it and clear everything if(framesSinceEvent > 5 && eventOcc){ // the > 5 makes sure to keep a small buffer between events int eventType = classifyevent(); switch(eventType){ //move case 0: fprintf(f,"A move as occurred\n"); break; //entry case 1: fprintf(f,"An entry as occurred\n"); break; //exit case 2: fprintf(f,"An exit as occurred\n"); break; default: fprintf(f,"An ERROR as occurred\n"); break; } clear(); framesSinceEvent = 0; eventOcc = false; cout << "Done classifying..." << endl; } if(eventOcc) framesSinceEvent++; } cb.unlock(prevImage); cb.unlock(currentImage); cvReleaseImageHeader(&cvCurrentImage); if(eventBufIndex <= 1){ cvReleaseImageHeader(&cvPrevImage); } else{ cvReleaseImage(&cvPrevImage); } cvReleaseImage(&cvFrameDiff); } } int DocumentTracker::kmeansCluster(){ int numPointsReassigned = 0; //initialize the centroids to the first NUM_CLUSTERS points for(int i = 0; i < NUM_CLUSTERS; i++){ centroids[i].ang = points[i].ang; centroids[i].hyp = points[i].hyp; centroids[i].id = i; } int stop = 1; int iter = 0; //converges when the centroid do not change (alternatively when all the points do not change clusters) while(stop > 0 && iter < 20){ stop = 0; numPointsReassigned = 0; //assign each point to its nearest centroid for(int i = 0; i < pointIndex; i++){ int clusterID = findClosestCentroid(points[i]); //a point changed clusters, so do not converge just yet if(clusterID != points[i].id){ stop = 1; numPointsReassigned++; } points[i].id = clusterID; } //after all points have been assigned recompute the centroids recomputeCentroids(); } return findLargestIndex(); } int DocumentTracker::findLargestIndex(){ int count = 0; int idToRet = 0; for(int i = 0; i < NUM_CLUSTERS; i++){ int tempcount = 0; for(int j = 0; j < pointIndex; j++){ if(points[j].id == i){ tempcount++; } } if(tempcount > count){ count = tempcount; idToRet = i; } } return idToRet; } int DocumentTracker::findClosestCentroid(cluster point){ double dist = MAXDOUBLE; int idToRet = 0; //find the closest centroid to point for(int i = 0; i < NUM_CLUSTERS; i++){ double temphyp = abs(centroids[i].hyp - point.hyp)*abs(centroids[i].hyp - point.hyp); double tempang = abs(centroids[i].ang - point.ang)*abs(centroids[i].ang - point.ang); double tempdist = sqrt(temphyp + tempang); if(tempdist < dist){ dist = tempdist; idToRet = i; } } return idToRet; } void DocumentTracker::recomputeCentroids(){ double hypsum = 0; double angsum = 0; int numPointsUsed = 0; int numCentroidsRecomputed = 0; //for every centroid take the mean of every point in its cluster for(int i = 0; i < NUM_CLUSTERS; i++){ for(int j = 0; j < pointIndex; j++){ if(points[j].id == i){ hypsum += points[j].hyp; angsum += points[j].ang; numPointsUsed++; } } if(numPointsUsed > 0){ hypsum = hypsum / numPointsUsed; angsum = angsum / numPointsUsed; centroids[i].ang = angsum; centroids[i].hyp = hypsum; numCentroidsRecomputed++; } hypsum = 0; angsum = 0; numPointsUsed = 0; } } int main(int argc, char **argv) { DocumentTracker dt; if(!dt.init()) cout << "Error initializing" << endl; if(!dt.start()) cout << "Error starting" << endl; dt.run(); dt.stop(); return 0; }