source: trunk/greenstone3-extensions/vishnu/src/cluster/procresults.c@ 8189

#include <jni.h>
#include <stdio.h>
#include <sys/types.h>
#include <unistd.h>
#include <sys/wait.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <math.h>
#include <ctype.h>
#include <sys/times.h>
#ifndef CLK_TCK
#include <time.h> 
#endif
//#include "Cluster.h"
#include "vishnu_server_Cluster.h"
#include "procresults.h"
#include "clustering.h"

char slash='/';


JNIEXPORT jstring JNICALL Java_vishnu_server_Cluster_getCC(JNIEnv * env,jobject thisObj, 
                               jintArray arr, jstring collection_home)
{
  fprintf(stderr,"in getCC");
  int i;
  jstring jresult;
  char * buffer = (char *) malloc(20000);
  const jbyte * coll_byte;
  char * coll_home;
  int docs[500];
  
  jsize len = (*env)->GetArrayLength(env, arr);
  jint *body = (*env)->GetIntArrayElements(env, arr, 0);
  
  coll_byte = (*env)->GetStringUTFChars(env, collection_home, NULL);
  if( coll_byte == NULL )
    {
      fprintf(stderr,"getCC: colleciton_home is null, returning null");
      return NULL;
    }
  coll_home = coll_byte;
  
  for( i = 0; i < len; i++ ) docs[i] = body[i];
  
  mediate(docs,len,buffer, coll_home);
  jresult = (*env)->NewStringUTF(env, buffer);
  
  (*env)->ReleaseStringUTFChars(env, collection_home, coll_byte);
  (*env)->ReleaseIntArrayElements(env,arr,body,0);
  return jresult;
}


void mediate(int* docs, int length, char *buffer, char *coll_home)
{
  work(docs,length,buffer,coll_home);
  return;
}


DocData *getIntVector(size_t docToRead,FILE *docTableFp,
              FILE *dtNdxFp, size_t maxDoc, size_t tableLen,
              size_t *recCount)
{
  size_t recSize, docInf,docNext,c;
  DocData *data;
  recSize=sizeof(size_t)+sizeof(short);
  
  fseek(dtNdxFp,(docToRead-1)*sizeof(size_t),SEEK_SET);
  fread(&docInf,sizeof(size_t),1,dtNdxFp);
  
  if(docToRead==1)
    docInf=0;
  
  fseek(docTableFp,docInf,SEEK_SET);
  
  if(docToRead<maxDoc)fread(&docNext,sizeof(size_t),1,dtNdxFp);
  else docNext=tableLen;
  
  
  
  *recCount=(docNext-docInf)/recSize;
  data=(DocData *)malloc(*recCount*sizeof(DocData));
  
  assert(data);
  for(c=0;c < *recCount;c++)
    {
      fread(&data[c],recSize,1,docTableFp);
      if(feof(docTableFp)) break;
    }
  return data;
}

void buildMatrix(TranslationItemPtr transTable,
         size_t arraySize,DocRec *allVectors,
         size_t vectorCount, MatrixNodePtr *matrix)
{
  MatrixNodePtr newNode;
  size_t c,d,e; 
  for(c=0;c<vectorCount;c++)
    {
      e=0;
      matrix[c]=NULL;
      for(d=0;d<allVectors[c].count && e < arraySize;d++)
    {
      
      while(allVectors[c].data[d].pos>transTable[e].itemPos && e < arraySize)
        {
          e++;
        }
      if(e < arraySize && allVectors[c].data[d].pos==transTable[e].itemPos)
        {
          /* this for straight text output 
         can't do this as some vectors are empty
         printf("%s%d,%d",matrix[c]==NULL?"":",",
         transTable[e].newColumn,allVectors[c].data[d].freq);
         this is for a jni interface? */
          newNode=malloc(sizeof(MatrixNode));
          assert(newNode);
          newNode->next=matrix[c];
          newNode->column=transTable[e].newColumn;
          newNode->freq=allVectors[c].data[d].freq;
          matrix[c]=newNode;
        }
      
      
    }
      //printf("\n");               
    }
}


int removeEmptyRows(MatrixNodePtr **matrixPtr, DocRec **allVectorsPtr,char ***descriptionsPtr,int vectorCount)
{
  DocRec *newAllVectors;
  MatrixNodePtr *newMatrix, current;
  char **newDescriptions;
  int c,d;
  
  newAllVectors = calloc(MAX_DOCUMENTS,sizeof(DocRec));
  assert(newAllVectors);
  newMatrix = calloc(MAX_DOCUMENTS,sizeof(MatrixNodePtr));
  assert(newMatrix);
  newDescriptions=calloc(MAX_DOCUMENTS,sizeof(char *));
  assert(newDescriptions);
  //    for(d=0,c=0;c<vectorCount;c++)
  //        fprintf(stderr,"current %p\n",(*matrixPtr)[c]);
  for(d=0,c=0;c<vectorCount;c++)
    {
      current=(*matrixPtr)[c];
      //        fprintf(stderr,"vector %d\n",c);
      fflush(stderr);
      if(current != NULL) 
    {
      //            fprintf(stderr,"current %p allvectors %p descriptions %p\n",current,(*allVectorsPtr)[c],(*descriptionsPtr)[c]);
      newMatrix[d] = current;
      newAllVectors[d] = (*allVectorsPtr)[c];
      newDescriptions[d] = (*descriptionsPtr)[c];
      
      //            fprintf(stderr,"copied to new vector %d\n",d);
      //            fflush(stderr);
      d++;
    }
      else
        {
      free((*allVectorsPtr)[c].data);
      free((*descriptionsPtr)[c]);
        }
    }
  free(*matrixPtr);
  free(*descriptionsPtr);
  free(*allVectorsPtr);
  *matrixPtr = newMatrix;
  *descriptionsPtr=newDescriptions;
  *allVectorsPtr=newAllVectors;
  return d;
}


int main(int argc, char **argv)
{
  return 0;
}


char * appendString(char * buffer, int * cursize, int * curptr, char *data, int len)
{
  if (!len) return (buffer);
  if (len + *curptr + 1 > *cursize) {
    fflush(stderr);
    while (len + *curptr + 1 > *cursize) *cursize += 256;
    buffer = (char *) realloc ((void *) buffer, *cursize);
  }
  strncpy (buffer + *curptr, data, len);
  *curptr += len;
  buffer[*curptr] = '\0';
  return (buffer);
}

void work(int *docNums, int numDocs, char * buffer, char *coll_home)
{
  time_t baseTime,wordListTime;
  struct tms *timer;
  
  DocRec *allVectors;
  InterestItem *itemArray;
  TranslationItem *transTable;
  MatrixNodePtr *matrix,current,last;
  char **descriptions;
  
  char bbuf[40];
  int cursize = 20000;
  int curptr = 0;
  
  int clusterNum = 10;
  int limit = 20;
  
  size_t arraySize;
  size_t maxDoc;
  FILE *index;
  FILE *data;
  FILE *words;
  FILE *binndx;
  FILE *freqs;
  char *dir, indexName[STRING_BUFF_LEN],dataName[STRING_BUFF_LEN],wordsName[STRING_BUFF_LEN],
    binndxName[STRING_BUFF_LEN],freqName[STRING_BUFF_LEN], buff[STRING_BUFF_LEN], 
    indexDir[STRING_BUFF_LEN];
  
  size_t  *freqList,wordCount,c;
  size_t n,vectorCount, offset;
  HjTreePtr tree;
  
  timer = (struct tms *) malloc(sizeof (struct tms));

  fprintf(stderr,"About to get into ck data\n");

  allVectors = calloc(MAX_DOCUMENTS,sizeof(DocRec));
  assert(allVectors);
  itemArray = calloc(MAX_IWORDS,sizeof(InterestItem));
  assert(itemArray);
  transTable= calloc(MAX_IWORDS,sizeof(TranslationItem));
  assert(transTable);
  matrix = calloc(MAX_DOCUMENTS,sizeof(MatrixNodePtr));
  assert(matrix);
  descriptions=calloc(MAX_DOCUMENTS,sizeof(char *));
  assert(descriptions);

  //dir = "\0";
  //dir = getenv("GSDL3HOME");
  //sprintf(indexDir,"%s%cweb/sites/localsite/collect/%s/index/ck_index",dir,slash,coll);
  sprintf(indexDir, "%sindex%cck_index", coll_home, slash);
  //dir = getenv("GSDL3HOME");
  //fprintf(stderr,dir);

  sprintf(indexName,"%s%cdoctable.ndx",indexDir,slash);
  sprintf(dataName,"%s%cdoctable.data",indexDir,slash);
  sprintf(wordsName,"%s%cnewinterest.txt",indexDir,slash);
  sprintf(binndxName,"%s%cnewbnndexFn.ndx",indexDir,slash);
  sprintf(freqName,"%s%cnewdocFreq.dat",indexDir,slash);

  fprintf(stderr,indexName);

  index = fopen(indexName,"rb");
  assert(index);
  data = fopen(dataName,"rb");
  assert(data);
  words=fopen(wordsName,"r");
  assert(words);
  binndx=fopen(binndxName,"rb");
  assert(binndx);
  freqs=fopen(freqName,"rb");
  assert(freqs);

  wordCount=fileSize(binndx)/sizeof(size_t);
  fread(&maxDoc,sizeof(size_t),1,index);
  freqList=malloc(wordCount*sizeof(size_t));
  assert(freqList);
    
  // reads at most wordCount elements of size size_t into freqList
  n=fread(freqList,sizeof(size_t),wordCount,freqs);
  if(n != wordCount) fprintf(stderr,"n %d wordCount %d\n",n,wordCount); 
  assert(n==wordCount);
  fclose(freqs);

  vectorCount=fillAllVectors(allVectors, docNums, numDocs, data,index,maxDoc, descriptions);

  // begin processing results
  baseTime= times (timer);
  tree=constructWordList(freqList, allVectors, vectorCount);
  wordListTime= times(timer);
    
  arraySize = fillInterestArray(tree, MAX_IWORDS, 0,itemArray);
  vapeTree(tree);

  for (c=0;c<arraySize;c++)
    {
      transTable[c].newColumn=c;
      transTable[c].itemPos=itemArray[c].itemPos;
    }

  qsort((void *)transTable,arraySize,sizeof(TranslationItem),cmpTransTable);
  buildMatrix( transTable, arraySize,allVectors,vectorCount, matrix);

  //remove empty rows
  fflush(stderr);
  vectorCount= removeEmptyRows(&matrix,&allVectors,&descriptions,vectorCount);
  fflush(stderr);

  *buffer = '\0';

  n = sprintf(bbuf,"%d\n",vectorCount); 
  buffer = appendString(buffer,&cursize,&curptr,bbuf,strlen(bbuf));

  for(c=0;c<vectorCount;c++)
    {
      n=sprintf(bbuf,"%d\n",allVectors[c].docNum);
      appendString(buffer,&cursize,&curptr,bbuf,strlen(bbuf));
    }
    
  fprintf(stderr,"Buffer: %s\n",buffer);
  // print interesting words
  n= sprintf(bbuf,"%d\n",arraySize);
  buffer = appendString(buffer,&cursize,&curptr,bbuf,strlen(bbuf));

  for(c=0;c<arraySize;c++)
    {
      fseek(binndx,itemArray[c].itemPos*sizeof(size_t),SEEK_SET);
      fread(&offset,sizeof(size_t),1,binndx);
      fseek(words,offset,SEEK_SET);
      fgets(buff,STRING_BUFF_LEN,words);
      buffer = appendString(buffer,&cursize,&curptr,buff,strlen(buff));
      //printf("%s",buff);
    }
    
  // print matrix of words in documents
  n= sprintf(bbuf,"%d\n",vectorCount);
  buffer = appendString(buffer,&cursize,&curptr,bbuf,strlen(bbuf));

  for(c=0;c<vectorCount;c++)
    {
      current=matrix[c];
      while(current) 
    {
      n=sprintf(bbuf,"%d,%d%s",current->freq,current->column,current->next?",":"");
      buffer = appendString(buffer,&cursize,&curptr,bbuf,strlen(bbuf));
      current=current->next;
    }
      buffer = appendString(buffer,&cursize,&curptr,"\n",1);
    }

  fflush(stdout);

  // calculate and print clusters and sammon mapping
  clusterNum =  vectorCount>20?10:vectorCount/2;

  fprintf(stderr,"Cluster number: %d\n",clusterNum);

  limit =  vectorCount>20?20:(vectorCount/2)+(vectorCount/4);
  
  if  (vectorCount > 6)
    {
      clustering(COMPLETE_LINKAGE,matrix,vectorCount,arraySize,1,clusterNum,limit,buffer,&cursize,&curptr);
    }
  else if (vectorCount > 0)
    {
      printf("1\n");
      for (c=0;c<vectorCount;c++){ 
    printf("%d%s",c,c>=(vectorCount-1)?"\n":",");
    fprintf(stderr,"%d%s",c,c>=(vectorCount-1)?"\n":",");
      }
      printf("1\n0.5,0.5\n");
    }
  else printf("0\n0\n");

  for(c=0;c<vectorCount;c++)
    {
      current=matrix[c];
      while(current) 
    {
      last=current;
      current=current->next;
      free(last);
    }
      free(descriptions[c]);
      free(allVectors[c].data);
    }

  fclose(words);
  fclose(index);
  fclose(binndx);
  fclose(data);

  free(allVectors);
  free(descriptions);
  free(itemArray);
  free(transTable);
  free(matrix);
  free(freqList);
  free(timer);
}


int cmpTransTable(const void *a,const void *b)
{
  TranslationItemPtr aT, bT;
  aT = (TranslationItemPtr) a;
  bT = (TranslationItemPtr) b;
  return aT->itemPos - bT->itemPos;
}


size_t getDocNum(FILE *fp, char *buff)
{
  char *p;
  int c;
  size_t result;
  do
    {
      c = fgetc(fp);
        
    } while (!isdigit(c)&& !feof(fp));

  p=buff;

  while (isdigit(c)&& !feof(fp))
    { 
      *(p++) = c;
      c = fgetc(fp);
    }
  *p=0;
  result= atoi(buff);
  if(result)
    {
      for (c=0;c<DESCRIPTION_SIZE && !feof(fp);c++)
    {
      buff[c]=fgetc(fp);
      if(buff[c]==2)
        {
          buff[c]=0;
          break;
        }

    }
    }
  return result;
}


size_t fileSize(FILE *fp)
{
  size_t len;
  fseek(fp,0,SEEK_END);
  len=ftell(fp);
  rewind(fp);
  return len;
}


size_t fillAllVectors(DocRec *allVectors, int* docs, int length /*FILE *inFp*/, FILE *docTableFp,
              FILE *dtNdxFp, size_t maxDoc,char **descrip)
{
  size_t tableLen,  recCount, vectorCount=0,docNum=1;
    
  tableLen=fileSize(docTableFp);

  while (docNum > 0 && vectorCount < length /*&& !feof(inFp)*/ && vectorCount < MAX_DOCUMENTS && maxDoc!=0)
    {
      descrip[vectorCount]=malloc(DESCRIPTION_SIZE);
      assert(descrip[vectorCount]);
      //docNum=getDocNum(inFp, descrip[vectorCount]);
      docNum = docs[vectorCount];
      if(docNum>0)
    {
      allVectors[vectorCount].data=getIntVector(docNum,docTableFp,
                            dtNdxFp,  maxDoc,  tableLen, &recCount);
        
      allVectors[vectorCount].count=recCount;
      allVectors[vectorCount].docNum=docNum;
      vectorCount++;
    }
      else free(descrip[vectorCount]);
    }
  //while(!feof(inFp)) fgetc(inFp);
  return vectorCount;
}


HjTreePtr constructWordList(size_t *freqList,DocRec *allVectors, size_t vectorSize)
{
  HjTreePtr tree=NULL,newTree=NULL;
  size_t dj, c,d, e, m,n,pos;

  for(c=0;c<vectorSize;c++)
    {
      n=allVectors[c].count;
      m=n/2;
      for(d=m,e=m-1;d<n;d++,e--)
    {

      //allVectors[c].data[d].pos;
      //this is an insane device to get a flatter tree from already ordered data by 
      //jumbling up the order that it is added to the tree

      if(e>0)
        {
          pos=allVectors[c].data[e].pos;
          dj=freqList[pos];
          tree=buildHjTree(tree,&(allVectors[c].data[e]), dj);                              

          pos=allVectors[c].data[d].pos;
          dj=freqList[pos];
          tree=buildHjTree(tree,&(allVectors[c].data[d]), dj);
        }
      if(e==0)
        {
          pos=allVectors[c].data[e].pos;
          dj=freqList[pos];
          tree=buildHjTree(tree,&(allVectors[c].data[e]), dj);

          pos=allVectors[c].data[d].pos;
          dj=freqList[pos];
          tree=buildHjTree(tree,&(allVectors[c].data[d]), dj);

          if( n % 2 == 1 )
        {
          pos=allVectors[c].data[++d].pos;
          dj=freqList[pos];
          tree=buildHjTree(tree,&(allVectors[c].data[d]), dj);
          break;
        }


        }

    }
    }

  // get candidate keyword weights
  calcInterestVal(tree,vectorSize);
  newTree=treeByInterestVal(&tree,newTree);
  return newTree;

}


HjTreePtr buildHjTree(HjTreePtr tree, DocData *item, size_t freq)
{
  HjTreePtr root=tree, /*lastNode,*/ *treePtr;
  treePtr = &tree;
  while(1)
    {
      if(*treePtr==NULL)
    {   
      *treePtr=malloc(sizeof(HjTree));
      assert (*treePtr);
      (*treePtr)->less=NULL;
      (*treePtr)->more=NULL;
      (*treePtr)->dj=freq;
      (*treePtr)->hj=1;
      (*treePtr)->pos=item->pos;
      (*treePtr)->interestVal=0.0;
      if (root==NULL) root=(*treePtr);
      tree = (*treePtr);
      break;
    }
      else
    {
      if(tree->pos==item->pos)
        {
          tree->hj++;
          break;
        }
      else if((*treePtr)->dj < freq)
        {
          treePtr = &(tree->more);
          tree = (*treePtr);
        }
      else      
        {
          treePtr= &(tree->less);
          tree = (*treePtr);
        }
    }
    }   
  return root;
}

void calcInterestVal(HjTreePtr tree,size_t vectorSize)
{
  size_t hj,dj;

  //fprintf(stderr, "calcInterestVal\n");
  while(tree)
    {
      hj=tree->hj;
      dj=tree->dj;
      if(tree->more) 
    calcInterestVal(tree->more, vectorSize);
      tree->interestVal=((double)hj/(double)dj)*hj*log((double)vectorSize/hj);
      tree=tree->less;
    }
}

HjTreePtr getLeaf(HjTreePtr *tree)
{
  HjTreePtr result;
  if((*tree)->more) return getLeaf(&((*tree)->more));
  if((*tree)->less) return getLeaf(&((*tree)->less));
  else result=*tree;
  *tree=NULL;
  return result;
}

HjTreePtr insertByInterestVal(HjTreePtr tree,HjTreePtr node)
{
  if(tree==NULL)
    tree=node;
  else if(tree->interestVal < node->interestVal)
    tree->more=insertByInterestVal(tree->more,node);
  else
    tree->less=insertByInterestVal(tree->less,node);
  return tree;
}

HjTreePtr treeByInterestVal(HjTreePtr *tree, HjTreePtr newTree)
{
  HjTreePtr node;
  while(*tree)
    {
      node=getLeaf(tree);
      if(node)newTree=insertByInterestVal(newTree, node);
    }
  return newTree;
}


size_t fillInterestArray(HjTreePtr tree, size_t max, size_t count,
             InterestItem *itemArray)
{
  while(tree)
    {
      if(count==max) return count;
      if(tree->more) 
    count=fillInterestArray(tree->more,max, count,itemArray );
      if(count==max) return count;
      itemArray[count].interestVal = tree->interestVal;
      itemArray[count].itemPos = tree->pos;
      itemArray[count].dj = tree->dj;
      count++;
      tree=tree->less;
    }
  return count;
}

/*
  size_t printTree(HjTreePtr tree, size_t max, size_t count,
  FILE *binFp,FILE *wordsFp)
  {
  while(tree)
  {
  if(count==max) return count;
  if(tree->more) 
  count=printTree(tree->more,max, count,binFp,wordsFp);
  if(count==max) return count;
  printTreeItem(tree,binFp,wordsFp);
  count++;
  tree=tree->less;
  }
  return count;
  }

  void printTreeItem(HjTreePtr tree,FILE *binFp,FILE *wordsFp)
  {
  char buff[STRING_BUFF_LEN];
  size_t offset;
  fseek(binFp,tree->pos*sizeof(size_t),SEEK_SET);
  fread(&offset,sizeof(size_t),1,binFp);
  fseek(wordsFp,offset,SEEK_SET);
  fgets(buff,STRING_BUFF_LEN,wordsFp);
  printf("%6d %6d %4.8f %s",tree->dj, tree->hj, tree->interestVal,buff);
  }
*/

void vapeTree(HjTreePtr tree)
{
  if(!tree)return;
  vapeTree(tree->more);
  vapeTree(tree->less);
  free(tree);
}