#include "patterns-pt.h"

struct pat_length *table;

#ifdef BIG_FILE
unsigned long file_bytes;
#else
unsigned int file_bytes;
#endif

int Verbose;
int frequency;
char *mem_file;
unsigned int **search_members;

static void progress_func(int max_searchs, int pat_len);
void *match_search(void *set);

static void progress_func(int max_searchs, int pat_len){
	static float a = 0;
	static float b = 0;

	if(max_searchs == 0){
		b++;
		printf("progress: %.02lf%% %d length patterns done.\n",(double)((b/a)*100),pat_len );
	}else{
		a = max_searchs;
		printf("progress: %.02lf%%\n",(double)((b/a)*100));
	}
}

void reset_index(int member_length, char *index){
	int num3=0;
	int num4=0;
	if(member_length>5){
		for(num3=0; num3<256; num3++){
			for(num4=0; num4<256; num4++){ 
				index[num3*256+num4]=0;
				index[65536+num3*256+num4]=0;
			}
		}
		return;
	}

	for(num4=0; num4<256; num4++){ 
		index[num3*256+num4]=0;
	}
	return;
}

static int Compare(register unsigned int num5, register unsigned int num6, register unsigned int saved){
			int num7 = num5+saved;
			while(num5<num7-7){
				if(mem_file[num6] != mem_file[num5])
					return(1);
				if(mem_file[num6+1] != mem_file[num5+1])
					return(1);
				if(mem_file[num6+2] != mem_file[num5+2])
					return(1);
				if(mem_file[num6+3] != mem_file[num5+3])
					return(1);
				if(mem_file[num6+4] != mem_file[num5+4])
					return(1);
				if(mem_file[num6+5] != mem_file[num5+5])
					return(1);
				if(mem_file[num6+6] != mem_file[num5+6])
					return(1);
				if(mem_file[num6+7] != mem_file[num5+7])
					return(1);


				num5+=8;
				num6+=8;
			}
			//saved-=7;
                        //while(num5>saved+3){
			while(num5<num7-3){
				//8 bytes

//				result = (mem_file[num6] != mem_file[num5] && mem_file[num6-1] != mem_file[num5-1] && mem_file[num6-2] != mem_file[num5-2] && mem_file[num6-3] != mem_file[num5-3]);
//				if(result)
//					return(1);


				if(mem_file[num6] != mem_file[num5])
					return(1);
				if(mem_file[num6+1] != mem_file[num5+1])
					return(1);
				if(mem_file[num6+2] != mem_file[num5+2])
					return(1);
				if(mem_file[num6+3] != mem_file[num5+3])
					return(1);


				num5+=4;
				num6+=4;

//                                if(mem_file[num6--] != mem_file[num5--])
//                                        return(1);
//                                if(mem_file[num6--] != mem_file[num5--])
//                                        return(1);
//                                if(mem_file[num6--] != mem_file[num5--])
//                                        return(1);
//                                if(mem_file[num6--] != mem_file[num5--])
//                                        return(1);


                        }
			//error with 2 lengths ...
			//if(saved < 3)
			//	return(1);

                        //saved-=3;
                        //while(num5>saved){
			while(num5<num7){
                                if( mem_file[num6++] != mem_file[num5++])
                                        return(1); 
                        }



                        return(0);
}

void *match_search(void *set){
        //returns 0 for match found and written
        //returns 1 for no matches found

	int num7;
	int num1;
	
#ifdef BIG_FILE
	unsigned long num4;  
	unsigned long num2=0;
	register unsigned long num3;
	unsigned long num5=0;
	unsigned long num6=0;
	unsigned long highest_times=0;
	const unsigned long stop_byte = file_bytes-table[num1].len-128;
	unsigned long pre_int=0;

	unsigned long repeat_found=0;
	unsigned long repeat_total=0;

	unsigned long table_miss = 0; 
	unsigned long table_found = 0; 
	unsigned long cache_found = 0;   
	unsigned long buffer_found = 0;  
	unsigned long patterns_total = 0;
	unsigned long patterns_freed = 0;
	unsigned long repeat_sub=0;
	if(Verbose){
		printf("Warning BIG_FILE is not thought out !\n");
		printf("Danger! don't run this !\n");
	}
#else
        unsigned int num4;
        unsigned int num2=0;
        register unsigned int num3;
	unsigned int num5=0;
	unsigned int num6=0;
	unsigned int highest_times=0;
	unsigned int stop_byte; //= file_bytes-table[num1].len-128;
	unsigned int pre_int=0;
	unsigned int repeat_found=0;
	unsigned int repeat_total=0;
	unsigned int table_miss = 0;
	unsigned int table_found = 0;
	unsigned int cache_found = 0;
	unsigned int buffer_found = 0;
	unsigned int patterns_total = 0;
	unsigned int patterns_freed = 0;
	unsigned int repeat_sub=0;
	unsigned int largest_id_size=0;
#endif

//	int id_ret=0;
	unsigned short member_length; // = table[num1].len-1;
	unsigned short search_offset = 0;

	unsigned int saved=0;
        int prefetch_a[128];
        int prefetch_b[8];

        unsigned int pre_count=0;
        int pre_ret=0;

	unsigned char first_char=0;
	unsigned char last_char=0;	
	unsigned char sec_char=0;
	unsigned char a_char=0;
	unsigned char b_char=0;
	unsigned char c_char=0;

	register unsigned int id = 0;

        char *index;
	unsigned int sizes[256][256];
	struct buffer_structure *buffer[256];
	struct common_structure *common[256];
	unsigned int buf_members[256];
	unsigned int buf_count[256];
	unsigned int com_count[256];

	unsigned int *unique_id[256];
	unsigned int *common_id[256];
	unsigned int *buffer_id[256];
	unsigned int tbl_mems[256];

	int *buffer_open[256];
	int *common_open[256];
	struct translation_structure *temp_struct_array;
	double cache_sizes[2];
	unsigned short b_members;
	unsigned short c_members;

	int tbl_count = 1;
	const short unsigned int thread = table[*(unsigned int *)set].search_num;
        struct timeval starttime,endtime;
        double time1;
        
	while(tbl_count <= search_members[thread][0]){ 
		gettimeofday(&starttime, NULL);
		num1=search_members[thread][tbl_count];
        	pre_int=0;     
        	repeat_found=0;
        	repeat_total=0;
        	table_miss = 0;
        	table_found = 0;
        	cache_found = 0;
        	buffer_found = 0;
        	patterns_total = 0;
        	patterns_freed = 0;
        	repeat_sub=0;
		pre_ret=0;
		pre_count=0;
		saved=0;

		member_length = table[num1].len-1;
		stop_byte = file_bytes-table[num1].len-128;
	
        	if(member_length>5){
                	index = malloc(sizeof(char)*65536*2);
        	}else{
                	index = malloc(sizeof(char)*65536);
        	}

		num3 = 256*(sizeof(struct buffer_structure)+sizeof(int)+sizeof(int)+member_length+1);
		//num4=floor(917504/num3);
		//num4=floor(786432/num3);
		num4=floor(655360/num3);
		//num4=floor(524288/num3);
		//num4=floor(393216/num3);
		//num4=floor(1572864/num3);
		//num4=floor(262144/num3);
		//num4=floor(131072/num3);
		//num4=floor(65536/num3);

		num3 = 256*(sizeof(struct common_structure)+sizeof(int)+member_length+1);
		//num5=floor(1572864/num3);
		//num5=floor(524288/num3);
		//num5=floor(655360/num3);
		//num5=floor(393216/num3);
		num5=floor(262144/num3);
		//num5=floor(196608/num3);
		//num5=floor(131072/num3);
		//num5=floor(65536/num3);


		if(num4 < 8){
			num4=8;
		}
		while(num4 % 8 != 0){
			num4++;
		}
		while(num5 % 4 != 0){
			num5++;
		}
		b_members = num4;
		c_members = num5;

		for(num3=0; num3<256; num3++){
			for(num4=0; num4<256; num4++){
				sizes[num3][num4]=1;
			}
			buffer[num3] = malloc(b_members*sizeof(struct buffer_structure));
			common[num3] = malloc(c_members*sizeof(struct common_structure));
			buffer_id[num3] = malloc(b_members*sizeof(unsigned int));
			common_id[num3] = malloc(c_members*sizeof(unsigned int));
			buffer_open[num3] = malloc(b_members*sizeof(int));
			common_open[num3] = malloc(c_members*sizeof(int));

			for(num4=0; num4<b_members; num4++){			
				buffer[num3][num4].string = malloc(table[num1].len*sizeof(char));
			}
			for(num4=0; num4<c_members; num4++){
				common[num3][num4].string = malloc(table[num1].len*sizeof(char));
			}
		}
		cache_sizes[0] = 256*b_members*( sizeof(struct buffer_structure)+sizeof(int)+sizeof(int)+(table[num1].len*sizeof(char)) );
		cache_sizes[1] = 256*c_members*( sizeof(struct common_structure)+sizeof(int)+(table[num1].len*sizeof(char)) );
		
		//first_char loop
		for(num7=255; num7>-1; num7--){
			num2=0;
			reset_index(member_length, index);
			for(num3=0; num3<256; num3++){
				buf_members[num3] = 0;
				buf_count[num3] = 0;
				com_count[num3]=0;
				tbl_mems[num3]=0;
				for(num4=0; num4<b_members; num4++){
					buffer_open[num3][num4]=1;
					buffer[num3][num4].times=0;
				}
				for(num4=0; num4<c_members; num4++){
					common[num3][num4].times=0;
					common[num3][num4].sub=0;
					common_open[num3][num4]=1;
				}
			
				unique_id[num3]=malloc(sizeof(unsigned int)*2);
				unique_id[num3][0]=0;
			}

			for(num3=0; num3<8; num3++){
				prefetch_b[num3] = num7;
			}
			first_char = num7;
			repeat_found=0;

			//entire file scan loop
			while(num2<stop_byte){
				//fill a page 16bytes at a time
				for(num3=0; num3<121; num3+=8){
					prefetch_a[num3] = (unsigned char)mem_file[num2+num3];
					prefetch_a[num3+1] = (unsigned char)mem_file[num2+num3+1];
					prefetch_a[num3+2] = (unsigned char)mem_file[num2+num3+2];
					prefetch_a[num3+3] = (unsigned char)mem_file[num2+num3+3];
					prefetch_a[num3+4] = (unsigned char)mem_file[num2+num3+4];
					prefetch_a[num3+5] = (unsigned char)mem_file[num2+num3+5];
					prefetch_a[num3+6] = (unsigned char)mem_file[num2+num3+6];
					prefetch_a[num3+7] = (unsigned char)mem_file[num2+num3+7];
				}
				//parse page for num7 matches
				for(num3=0; num3<121; num3+=8){
					if(prefetch_a[num3] == prefetch_b[0]){
						prefetch_a[num3]=-1;
					}
					if(prefetch_a[num3+1] == prefetch_b[1]){
						prefetch_a[num3+1]=-1;
					}
					if(prefetch_a[num3+2] == prefetch_b[2]){
						prefetch_a[num3+2]=-1;
					}
					if(prefetch_a[num3+3] == prefetch_b[3]){
						prefetch_a[num3+3]=-1;
					}
					if(prefetch_a[num3+4] == prefetch_b[4]){
						prefetch_a[num3+4]=-1;
					}
					if(prefetch_a[num3+5] == prefetch_b[5]){
						prefetch_a[num3+5]=-1;
					}
					if(prefetch_a[num3+6] == prefetch_b[6]){
						prefetch_a[num3+6]=-1;
					}
					if(prefetch_a[num3+7] == prefetch_b[7]){
						prefetch_a[num3+7]=-1;
					}
				}
				//run the matches
				pre_int=num2;
				pre_count=0;
				while(pre_count<125){
					if(prefetch_a[pre_count] == -1){
						num2=pre_int+pre_count;
						pre_ret=1;
						goto proceed;
					}
					pre_a:
					pre_count++;
					if(prefetch_a[pre_count] == -1){
						num2=pre_int+pre_count;
						pre_ret=2;
						goto proceed;
					}
					pre_b:
					pre_count++;
					if(prefetch_a[pre_count] == -1){
						num2=pre_int+pre_count;
						pre_ret=3;
						goto proceed;
					}
					pre_c:
					pre_count++;
					if(prefetch_a[pre_count] == -1){
						num2=pre_int+pre_count;
						pre_ret=4;
						goto proceed;
					}
					pre_d:
					pre_count++;
				}
				goto found;
				proceed:
			
				sec_char = (unsigned char)mem_file[num2+1];
				last_char = (unsigned char)mem_file[num2+member_length];
				//a_char = (unsigned char)mem_file[num2+member_length-1];

				if(member_length>5){

					a_char = (unsigned char)mem_file[num2+member_length-1];
					b_char = (unsigned char)mem_file[num2+member_length-2];
					search_offset = member_length-3;
					c_char = (unsigned char)mem_file[num2+search_offset];
					id = c_char*16777216 + b_char*65536 + a_char*256 + last_char;
				}else{
					search_offset = member_length-1;

					a_char = (unsigned char)mem_file[num2+search_offset];
					id = first_char*16777216 + sec_char*65536 + a_char*256 + last_char;
				}


				goto repeat_search;

				buffer_search:
                        	num3=0;
                        	//search cached xMB worth of most recent found patterns
                        	while(num3<b_members){
                                	if(!buffer_open[last_char][num3]){
						if(id != buffer_id[last_char][num3]){
							goto r_next;
						}
						for( num4=1; num4 < search_offset; num4++){
							if( mem_file[num2+num4] != buffer[last_char][num3].string[num4] ){
								goto r_next;
							}
						}
						buffer_found++;
						buffer[last_char][num3].times++;
	                                        if(pre_ret == 1){
        	                                        goto pre_a;
                	                        }
                        	                if(pre_ret == 2){
                                	                goto pre_b;
                                        	}
                                        	if(pre_ret == 3){
                                                	goto pre_c;
                                        	}
                                                goto pre_d;
					}
					r_next:
					num3++;
				}

				//search cached xMB worth of most often found patterns
				num3=0; 
				while(num3<c_members){
					if(!common_open[last_char][num3]){
						if(id != common_id[last_char][num3]){
							goto o_next;
						}
						for( num4=1; num4 < search_offset; num4++){
							if( mem_file[num2+num4] != common[last_char][num3].string[num4] ){
								goto o_next;						
							}
						}
						cache_found++;
						common[last_char][num3].times++;
	                                        if(pre_ret == 1){
        	                                        goto pre_a;
                	                        }
                        	                if(pre_ret == 2){
                                	                goto pre_b;
                                        	}
                                        	if(pre_ret == 3){
                                                	goto pre_c;
	                                        }
                                                goto pre_d;
					}
					o_next:
					num3++;
				}

				//table search
				num3 = unique_id[last_char][0];
				//num3 = tbl_mems[last_char];
				//num3 = table[num1].member[first_char][last_char][0].times;
				while(num3>0){
					//saved=table[num1].member[first_char][last_char][num3].series;
					//if(unique_id[last_char][num3] == id){
					//id = first_char*16777216 + sec_char*65536 + a_char*256 + last_char;
					//id = c_char*16777216 + b_char*65536 + a_char*256 + last_char;

					//sec_char
					//if(mem_file[saved+1] == sec_char){
						//last_char
						//if(mem_file[saved+member_length] == last_char){
							//a_char
							//if(mem_file[saved+member_length-1] == a_char){
								//saved++;
								//saved=table[num1].member[first_char][last_char][num3].series;
							if(id == unique_id[last_char][num3]){
								//if( Compare(saved+member_length-2, num2+member_length-2, saved) ){
								saved=table[num1].member[first_char][last_char][num3].series; 
								if( Compare(saved+1, num2+1, search_offset) ){
									goto next5;
								}
								goto id_match;
							}
						//}
					//}
					next5:
					num3--;
				}
				table_miss++;
				goto no_search;

				id_match:
				table[num1].member[first_char][last_char][num3].times++;
				//tbl_mems[last_char]++;

				for(num4=com_count[last_char]; num4<c_members; num4++){
					if(table[num1].member[first_char][last_char][num3].times>common[last_char][num4].times){
						//contribute old results to table before replacing
						if(!common_open[last_char][num4]){
							num6 = common[last_char][num4].sub;
							table[num1].member[first_char][last_char][num6].times=common[last_char][num4].times;
						}
						//write new results to common index
						common_open[last_char][num4]=0;
						common[last_char][num4].times=table[num1].member[first_char][last_char][num3].times;
						common_id[last_char][num4]=id;
						common[last_char][num4].sub=num3;
						for(num6=0; num6<member_length+1; num6++){
							common[last_char][num4].string[num6]=mem_file[table[num1].member[first_char][last_char][num3].series+num6];
						}
						if(com_count[last_char] == c_members-1){
							com_count[last_char] = 0;
						}else{
							com_count[last_char]++;
						}
						table_found++;
						if(pre_ret == 1){
							goto pre_a;
						}
						if(pre_ret == 2){
							goto pre_b;
						}
						if(pre_ret == 3){
							goto pre_c;
						}
						goto pre_d;
					}
				}
				table_found++;
				if(pre_ret == 1){
					goto pre_a;
				}
				if(pre_ret == 2){
					goto pre_b;
				}
				if(pre_ret == 3){
					goto pre_c;
				}
				goto pre_d;
		
				repeat_search:
				;
				if(first_char != (unsigned char)mem_file[num2+member_length]){
					if(member_length>5){
						if(!index[sec_char*256+last_char])
							goto no_search;
						if(!index[65536+a_char*256+b_char])
							goto no_search;
						goto buffer_search;
					}
					if(!index[sec_char*256+last_char])
						goto no_search;
					goto buffer_search;
				}
				//check if it's a repeat
				for(num3=1; num3<member_length; num3++){
					if(first_char != (unsigned char)mem_file[num2+num3]){
						if(member_length>5){
							if(!index[sec_char*256+last_char]) 
								goto no_search;
							if(!index[65536+a_char*256+b_char])
								goto no_search;
							goto buffer_search;
						}
						if(!index[sec_char*256+last_char])
							goto no_search;
						goto buffer_search;
					}
				}
				repeat_found++;
				if(repeat_found == 1)
					repeat_sub = num2;
	                        if(pre_ret == 1){
        	                     goto pre_a;
                	        }
                        	if(pre_ret == 2){
	                            goto pre_b;
        	                }
                	        if(pre_ret == 3){   
                        	    goto pre_c;
                        	}
				goto pre_d;
			
				no_search:
				;
				if(buf_members[last_char]<b_members){				
					for(num3=0; num3<member_length+1; num3++){
						buffer[last_char][ buf_count[last_char] ].string[num3]=mem_file[num2+num3];
					}
					buffer[last_char][ buf_count[last_char] ].series=num2;
					buffer_id[last_char][ buf_count[last_char] ] = id;
					buffer[last_char][ buf_count[last_char] ].times = 1;
					buffer_open[last_char][buf_count[last_char]]=0;

					if(member_length>5){
						index[sec_char*256+last_char]=1;
						index[65536+a_char*256+b_char]=1;
					}else{
						index[sec_char*256+last_char]=1;
					}
					buf_count[last_char]++;
					buf_members[last_char]++;
				}else{
					//take 4 oldest out of buffer and place into table
					//dealing with sets of 4
					if(buf_count[last_char] == b_members){
						buf_count[last_char] = 0;
					}

					for(num3=buf_count[last_char]; num3<buf_count[last_char]+4; num3++){
						num5 = table[num1].member[first_char][last_char][0].times;
						//num5 = tbl_mems[last_char];
						if(sizes[first_char][last_char] == num5){
							table[num1].member[first_char][last_char]=realloc(table[num1].member[first_char][last_char],sizeof(struct translation_structure) * (sizes[first_char][last_char]+1+3) );
							unique_id[last_char]=realloc(unique_id[last_char], sizeof(unsigned int) * (sizes[first_char][last_char]+1+3) );
							sizes[first_char][last_char]+=3;
						}					
						
						table[num1].member[first_char][last_char][num5+1].series = buffer[last_char][num3].series;
						table[num1].member[first_char][last_char][num5+1].times = buffer[last_char][num3].times;
						table[num1].member[first_char][last_char][0].times++;
						//tbl_mems[last_char]++;
						unique_id[last_char][ num5+1 ] = buffer_id[last_char][num3];
						unique_id[last_char][0] = table[num1].member[first_char][last_char][0].times;
						buffer_open[last_char][num3]=1;

						//add to common index if popular...
						for(num4=com_count[last_char]; num4<c_members; num4++){
							if(buffer[last_char][num3].times > common[last_char][num4].times){
								//contribute old results to table before replacing
								if(!common_open[last_char][num4]){
									num6 = common[last_char][num4].sub;
									//common[last_char][num4].string[member_length];
									table[num1].member[first_char][last_char][num6].times=common[last_char][num4].times;
								}
								//write new results to common index
								common_open[last_char][num4]=0;
								common[last_char][num4].times=buffer[last_char][num3].times;
								common_id[last_char][num4]=buffer_id[last_char][num3];
								common[last_char][num4].sub=num5+1;
								for(num6=0; num6<member_length+1; num6++){
									common[last_char][num4].string[num6]=buffer[last_char][num3].string[num6];
								}
								if(com_count[last_char] == c_members-1){
									com_count[last_char] = 0;
								}else{
									com_count[last_char]++;
								}
							}
						}
					}

					//put current pattern into buffer
					for(num3=0; num3<member_length+1; num3++){
						buffer[last_char][buf_count[last_char]].string[num3]=mem_file[num2+num3];
					}
					buffer[last_char][buf_count[last_char]].series=num2;
					buffer_id[last_char][buf_count[last_char]] = id;
					buffer[last_char][buf_count[last_char]].times = 1;
					buffer_open[last_char][buf_count[last_char]]=0;

					if(member_length>5){
						index[sec_char*256+last_char]=1;
						index[65536+a_char*256+b_char]=1;
					}else{
						index[sec_char*256+last_char]=1;
					}
					buf_count[last_char]++;
					buf_members[last_char]-=3;
				}
				found:
				num2=pre_int+pre_count;
			}
		
			if(repeat_found != 0){
				repeat_total += repeat_found;
				num5 = table[num1].member[first_char][first_char][0].times;
				if(sizes[first_char][first_char] == num5){
					table[num1].member[first_char][first_char]=realloc(table[num1].member[first_char][first_char], sizeof(struct translation_structure) * (sizes[first_char][first_char]+1+3) );
					sizes[first_char][first_char]+=3;
				}
				table[num1].member[first_char][first_char][0].times++;
				tbl_mems[last_char]++;
				table[num1].member[first_char][first_char][num5+1].times=repeat_found;
				table[num1].member[first_char][first_char][num5+1].series=repeat_sub;
				if(repeat_found > highest_times){
					highest_times=repeat_found;
				} 
			}


			num4 = largest_id_size;
			for(num3=0; num3<256; num3++){
				largest_id_size += sizes[first_char][num3]+1;
				free(unique_id[num3]);
			}
			if(num4>largest_id_size)
				largest_id_size=num4;



			//save all needed index info to tables before reset
			for(num3=0; num3<256; num3++){
				for(num6=0; num6<b_members; num6++){
					if(!buffer_open[num3][num6]){
						num5 = table[num1].member[first_char][num3][0].times;
						if(sizes[first_char][num3] == num5){					
							table[num1].member[first_char][num3]=realloc(table[num1].member[first_char][num3], sizeof(struct translation_structure) * (sizes[first_char][num3]+1+3) );
							sizes[first_char][num3]+=3;
						}
						table[num1].member[first_char][num3][0].times++;
						tbl_mems[last_char]++;
						table[num1].member[first_char][num3][num5+1].series=buffer[num3][num6].series;
						table[num1].member[first_char][num3][num5+1].times=buffer[num3][num6].times;
					}
				}

				for(num4=0; num4<c_members; num4++){
					if(!common_open[num3][num4]){
						num6 = common[num3][num4].sub;
						table[num1].member[first_char][num3][num6].times=common[num3][num4].times;
					}
				}
			}

			//free all strings that don't meet min frequency to save memory later
			for(num3=0; num3<256; num3++){
				num5=table[num1].member[first_char][num3][0].times+1;
				num6=1;
				temp_struct_array = malloc(sizeof(struct translation_structure)*num5 );
				temp_struct_array[0].times = table[num1].member[first_char][num3][0].times;
				temp_struct_array[0].series=0;
				for(num4=1; num4<num5; num4++){
					if(table[num1].member[first_char][num3][num4].times>frequency-1){
						if(table[num1].member[first_char][num3][num4].times>highest_times){
							highest_times = table[num1].member[first_char][num3][num4].times;
						}
						temp_struct_array[num6].series=table[num1].member[first_char][num3][num4].series;
						temp_struct_array[num6].times=table[num1].member[first_char][num3][num4].times;
						num6++;
					}else{
						temp_struct_array[0].times--;
					}
				}
				patterns_freed+=sizes[first_char][num3]+1-num6;
				free(table[num1].member[first_char][num3]);
				table[num1].member[first_char][num3]=malloc(sizeof(struct translation_structure)*num6 );
				for(num4=0; num4<num6; num4++){
					table[num1].member[first_char][num3][num4].series=temp_struct_array[num4].series;
					table[num1].member[first_char][num3][num4].times=temp_struct_array[num4].times;
				}
			}
		}

		table[num1].highest_times=highest_times;
		if(Verbose){
			progress_func(0, member_length+1);
			printf("\tbuffer/cache sizes\n");
			printf("\t\trecent: %u(%.02lfKB)", b_members, cache_sizes[0]/1024); 
			printf(" common: %u(%.02lfKB)", c_members, cache_sizes[1]/1024); 

			if(member_length>5){
				printf(" index: %.02lfKB", (double)sizeof(char)*256*256*2/1024);
			}else{
				printf(" index: %.02lfKB", (double)sizeof(char)*256*256/1024);
			}
		//printf(" unique_sec: %.02lfKB 
			printf(" unique_id: %.02lfKB\n", (double)largest_id_size*sizeof(int)/1024 );
		}
		//b_members, cache_sizes[0]/1024, c_members, cache_sizes[1]/1024, (double)sizeof(char)*256*256*2/1024, (double)largest_id_size*sizeof(unsigned char)/1024 ,(double)256*sizeof(unsigned int)/1024); 
		largest_id_size=0;

		patterns_total = table_found + buffer_found + cache_found + table_miss + repeat_total;
		if(patterns_total == 0){
			printf("WARING: Exiting to prevent floating-point exception!\n");
			printf("\tNo patterns found.\n");
			exit(1);
		}
		if(Verbose){
			printf("\ttotal patterns found: %u\n", patterns_total-table_miss);
			printf("\ttable_miss: %u table_found: %u recent_found: %u common_found: %u repeat_found: %u\n", table_miss, table_found, buffer_found, cache_found, repeat_total);
			printf("\ttable_miss: %.02lf%% table_found: %.02lf%% recent_found: %.02lf%% common_found: %.02lf%% repeat_found: %.02lf%%\n", (double)table_miss/patterns_total*100, (double)table_found/patterns_total*100, (double)buffer_found/patterns_total*100, (double)cache_found/patterns_total*100, (double)repeat_total/patterns_total*100 ); 

			printf("\tfreed: %u (%.04lfMB) low freq pattern structures\n", patterns_freed, (double)(patterns_freed*sizeof(struct translation_structure))/1024/1024);
		}
		for(num3=0; num3<256; num3++){
			//free(unique_id[num3]);
			free(buffer[num3]);
			free(common[num3]);
			free(common_id[num3]);
			free(buffer_id[num3]);
			free(buffer_open[num3]);
			free(common_open[num3]);
		}
		free(index);
		tbl_count++;

		gettimeofday(&endtime, NULL);
		time1=((double)(endtime.tv_sec*1000000-starttime.tv_sec*1000000+endtime.tv_usec-starttime.tv_usec))/1000000;		
		if(Verbose)
			printf("\tHighest times: %u Pattern scan Rate: %.02lfKB/s\n", highest_times, (double)file_bytes/1024/time1);

	}
	pthread_exit(0);
}

//void usage(void){
//        printf("\tUsage: patterns <output> <input> <min pattern size (bytes)> <max pattern size (bytes)> <threads>\n");
//        exit(1);
//}


void Patterns_PT(int verbose, int thread_count, double *results){
	//int main(int argc, char **argv){
	//FILE *output;
	int input;
        struct stat buffer;
        int status;
        int pat_size = 4;
        int num1, num2, num3, num4, num5;
        //int progress=0;
	//double progress2=0;
        int max_searchs=0;
        int max_pat = 16;
        int half_file;
        frequency=4;
	//int thread_count = threads;
	struct timeval starttime,endtime;
	double time1;
	Verbose = verbose;
        //if(argc != 6){
        //        usage();
        //}

		//thread_count = atoi(argv[5]);
       // pat_size = atoi(argv[3]);
        //max_pat = atoi(argv[4]);

        input = open("patterns.data", O_RDONLY);
        if(input == -1){
                fprintf( stderr, "Error opening ./patterns.data\n");
                exit( 1 );
        }
	
        status = fstat(input, &buffer);
        file_bytes=buffer.st_size;
	if(file_bytes < max_pat+128){    
		printf("Error: file is too small!\n");
		exit(1);
	}
        
#ifndef BIG_FILE
	if(file_bytes > 4294967296-max_pat){
		printf("Error: this file exceeds 4GB, recompile with -D BIG_FILE\n");
		exit(1);
	}
#endif
 
       mem_file=(char *)mmap(0, file_bytes, PROT_READ, MAP_SHARED, input, 0);
        if (mem_file == MAP_FAILED) {
                close(input);
                perror("Error mmapping the file");
                exit(EXIT_FAILURE);
        }

		/*
        if( ( output = fopen( argv[1], "w" ) ) == NULL ) {
                fprintf( stderr, "Error opening %s\n", argv[1] );
                exit( 1 );
        }
		*/
		if(verbose){
			printf("input: ./patterns.data\n");
			//printf("output: %s\n", argv[1]);
			//printf("pat_size: %d\n", pat_size);
			printf("bytes: %d\n", file_bytes);
		}
        if(max_pat > file_bytes/2){
                max_pat = floor(file_bytes/2);
			if(verbose)
                printf("max_pat: %d (reduced to limit!)\n", max_pat);
        }else{
			if(verbose)
                printf("max_pat: %d\n", max_pat);
        }



        half_file = sizeof(unsigned char)*ceil(file_bytes/2);

        for(num1=pat_size; num1<max_pat; num1++){
                max_searchs++;
        }

	max_searchs++;
	if(verbose){
        printf("max_searchs: %d\n", max_searchs);
        printf("frequency: %d\n", frequency);
	}
if(thread_count > max_searchs){
	thread_count = max_searchs;
	if(verbose)
		printf("threads: %d(reduced!)\n", thread_count);
}else{
	if(verbose)
		printf("threads: %d\n", thread_count);
}
	pthread_t *threads = (pthread_t *)malloc(sizeof(pthread_t) * thread_count);

	table=(struct pat_length *)malloc( sizeof(struct pat_length) * max_searchs);

	for(num1=0; num1<max_searchs; num1++){
		table[num1].member=malloc( 256*sizeof(void *) );
		for(num2=0; num2<256 ; num2++){
                        table[num1].member[num2]=malloc( 256*sizeof(void *) );
				for(num3=0; num3<256; num3++){
					table[num1].member[num2][num3]=malloc( 2*sizeof(struct translation_structure) );
					table[num1].member[num2][num3][0].times=0;
				}
                }
        }




//unsigned int **search_members;
	search_members = malloc(sizeof(unsigned int *) * thread_count);
	num2 = ceil(max_searchs/thread_count);
	for(num1=0; num1<thread_count; num1++){
		search_members[num1]=malloc(sizeof(int) * num2);
		search_members[num1][0] = 0;
	}

	num2=0;
	for(num1=0; num1<max_searchs; num1++){
		table[num1].len=pat_size+num1;
		table[num1].search_num=num1;
		table[num1].highest_times=0;
		search_members[num2][0]++;
		search_members[num2][ search_members[num2][0] ]=num1;
		if(num2 == thread_count-1){
			num2=0;
		}else{
			num2++;
		}

	}


//////////////////////////////////


	//progress=0;
	//progress2=0;
	if(Verbose)
		progress_func(max_searchs, 0);

	gettimeofday(&starttime, NULL);

	for(num1=0; num1<thread_count; num1++){
		if (pthread_create(&threads[num1], NULL, match_search, &search_members[num1][1]) != 0)
			perror("pthread_create"), exit(1);
	}
	for(num1=0; num1<thread_count; num1++){
		if (pthread_join(threads[num1], NULL) != 0)
			perror("pthread_join"),exit(1);
	}


/*
	num2=1;
	for(num1=0; num1<max_searchs; num1++){
		table[num1].len=pat_size+num1;		
		table[num1].search_num=num1;
		table[num1].highest_times=0;

               	if (pthread_create(&threads[num1], NULL, match_search, &table[num1].search_num) != 0)
                       	perror("pthread_create"), exit(1);
				
		if(num2 % thread_count == 0 || num2 == max_searchs){
			for(num3=progress; num3<num2; num3++){
				if (pthread_join(threads[num3], NULL) != 0)
					perror("pthread_join"),exit(1);
			}
			progress=num2;
		}
		num2++;
	}
*/

	gettimeofday(&endtime, NULL);
	time1=((double)(endtime.tv_sec*1000000-starttime.tv_sec*1000000+endtime.tv_usec-starttime.tv_usec))/1000000;
	if(verbose)
		printf("time: %.02lfs\n", time1);


	//printf("generating output file...\n");
	/*
	gettimeofday(&starttime, NULL);

        //parse structures and write to output

	int index1=0;
	int index2=0;
	int max_print=0;
	int num6;
        struct sort_structure *sort_index;
	sort_index = malloc(max_searchs*sizeof(struct sort_structure));

	//initialize sort_index to all 0's
	for(num5=0; num5<max_searchs; num5++){
		sort_index[num5].highest=0;
		for(num6=0; num6<256; num6++){
			for(index1=0; index1<256; index1++){
				sort_index[num5].sub_highest[num6][index1]=0;
			}
		}
	}
	
	//search tables for the one containing the highest count
	num2=0;
	for(num1=0; num1<max_searchs; num1++){
		//find highest member count
		sort_index[num1].highest=table[num1].highest_times;
		if(sort_index[num1].highest > num2){
			num2=sort_index[num1].highest;
		}
	}

	//search tables for highest in all sets and save to sort_index
	for(num1=max_searchs-1; num1>-1; num1--){
		for(index1=0; index1<256; index1++){
			for(index2=0; index2<256; index2++){
				num5=table[num1].member[index1][index2][0].times+1;
				for(num4=1; num4<num5; num4++){
					if(table[num1].member[index1][index2][num4].times > sort_index[num1].sub_highest[index1][index2]){
						sort_index[num1].sub_highest[index1][index2]=table[num1].member[index1][index2][num4].times;
					}
				}
			}
		}
	}

	while(num2>=frequency){
		for(num1=max_searchs-1; num1>-1; num1--){
			if(sort_index[num1].highest == num2){
				sort_index[num1].highest = 0;
				for(index1=0; index1<256; index1++){
					for(index2=0; index2<256; index2++){
						if(sort_index[num1].sub_highest[index1][index2] == num2){
							sort_index[num1].sub_highest[index1][index2]=0;
							num5=table[num1].member[index1][index2][0].times+1;
							for(num4=1; num4<num5; num4++){
								if(table[num1].member[index1][index2][num4].times == num2){			
									fprintf(output, "%d %d", table[num1].len, table[num1].member[index1][index2][num4].times);
                        						for(num3=0; num3<table[num1].len; num3++){
										fprintf(output, " %d", mem_file[table[num1].member[index1][index2][num4].series+num3]);
                        						}
									fprintf(output, "\n");
									max_print++;
									if(max_print == 40000){
										goto complete;
									}
                        					}else if(table[num1].member[index1][index2][num4].times < num2){
									if(table[num1].member[index1][index2][num4].times > sort_index[num1].sub_highest[index1][index2]){
										sort_index[num1].sub_highest[index1][index2] = table[num1].member[index1][index2][num4].times;
									}
								}
							}
						}
						if(sort_index[num1].sub_highest[index1][index2] > sort_index[num1].highest){
							sort_index[num1].highest=sort_index[num1].sub_highest[index1][index2];
						}
					}
				}
			}
		}
				
		num2=0;
		for(num6=0; num6<max_searchs; num6++){
			//find highest member count
			if(sort_index[num6].highest > num2){
				num2=sort_index[num6].highest;
			}
		}
        }

	complete:
	;

	gettimeofday(&endtime, NULL);


	time1=((double)(endtime.tv_sec*1000000-starttime.tv_sec*1000000+endtime.tv_usec-starttime.tv_usec))/1000000;
	printf("time: %.02lfs\n", time1);
	*/

        if (munmap(mem_file, file_bytes) == -1) {
                perror("Error un-mmapping the file");
        }

	//free(mem_file);
        close(input);

    //    fclose(output);
	if(verbose)
		printf("done.\n");
	results[1] = (double)file_bytes/1024/time1;
	results[1] *= max_searchs;
	results[0] += results[1];
        //return 0;
}