#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <stdint.h>
#include <unistd.h>
#include <sys/mman.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <pthread.h>
#include <time.h>
#include <sys/time.h>
#include "patterns-pt.h"


struct sort_structure{
	uint32_t highest;
	uint32_t sub_highest[256][256];
};

struct translation_structure{
#ifdef BIG_FILE
	uint64_t series;
#else
	uint32_t series;
#endif
	uint32_t times;
};

struct pat_length{
	uint32_t highest_times;
	uint32_t len;
	uint32_t search_num;
	struct translation_structure ***member;
}*table;

struct buffer_structure {
	uint32_t times;
	char *string;
#ifdef BIG_FILE
        uint64_t series;
#else
        uint32_t series;
#endif
};

struct common_structure {
	uint32_t times;
	char *string;
	uint32_t sub;
};

#ifdef BIG_FILE
uint64_t file_bytes;
#else
uint32_t file_bytes;
#endif

int Verbose;
uint32_t frequency;
char *mem_file;
uint32_t **search_members;
int *thread_id;

static void progress_func(uint32_t max_searchs, uint32_t pat_len);
void *match_search(void *set);

static void progress_func(uint32_t max_searchs, uint32_t pat_len){
	static float a = 0;
	static float b = 0;

	if(max_searchs == 0){
		b++;
		printf("progress: %.02f%% %u length patterns done.\n", (b/a)*100, pat_len);
	}else{
		a = max_searchs;
		printf("progress: %.02f%%\n",(b/a)*100);
	}
}

void reset_index(uint32_t member_length, char *index){
	uint32_t num3=0;
	uint32_t num4=0;
	if(member_length>5){
		for(num3=0; num3<256; num3++){
			for(num4=0; num4<256; num4++){ 
				index[num3*256+num4]=0;
				index[65536+num3*256+num4]=0;
			}
		}
		return;
	}

	for(num4=0; num4<256; num4++){ 
		index[num3*256+num4]=0;
	}
	return;
}

static int Compare(uint32_t num5, uint32_t num6, uint32_t saved){
	uint32_t num7 = num5+saved;
	while(num5<num7-7){
		if(mem_file[num6] != mem_file[num5])
			return(1);
		if(mem_file[num6+1] != mem_file[num5+1])
			return(1);
		if(mem_file[num6+2] != mem_file[num5+2])
			return(1);
		if(mem_file[num6+3] != mem_file[num5+3])
			return(1);
		if(mem_file[num6+4] != mem_file[num5+4])
			return(1);
		if(mem_file[num6+5] != mem_file[num5+5])
			return(1);
		if(mem_file[num6+6] != mem_file[num5+6])
			return(1);
		if(mem_file[num6+7] != mem_file[num5+7])
			return(1);
		num5+=8;
		num6+=8;
	}
	while(num5<num7-3){
		if(mem_file[num6] != mem_file[num5])
			return(1);
		if(mem_file[num6+1] != mem_file[num5+1])
			return(1);
		if(mem_file[num6+2] != mem_file[num5+2])
			return(1);
		if(mem_file[num6+3] != mem_file[num5+3])
			return(1);
		num5+=4;
		num6+=4;
	}
	while(num5<num7){
		if( mem_file[num6++] != mem_file[num5++])
			return(1);
	}
	return(0);
}

void *match_search(void *set){
	//returns 0 for match found and written
	//returns 1 for no matches found
	int num7;
	int num1;
	//printf("here 1 ... %u\n", *(uint32_t *)set);
#ifdef BIG_FILE
	uint64_t num4;
	uint64_t num2=0;
	uint64_t num3;
	uint64_t num5=0;
	uint64_t num6=0;
	uint64_t highest_times=0;
	const uint64_t stop_byte = file_bytes-table[num1].len-128;
	uint64_t pre_int=0;
	uint64_t repeat_found=0;
	uint64_t repeat_total=0;
	uint64_t table_miss = 0;
	uint64_t table_found = 0;
	uint64_t cache_found = 0;
	uint64_t buffer_found = 0;
	uint64_t patterns_total = 0;
	uint64_t patterns_freed = 0;
	uint64_t repeat_sub=0;
	printf("Warning BIG_FILE is not thought out !\n");
	printf("Danger! don't run this !\n");
#else
	uint32_t num4=0;
	uint32_t num2=0;
	uint32_t num3=0;
	uint32_t num5=0;
	uint32_t num6=0;
	uint32_t highest_times=0;
	uint32_t stop_byte=0;
	uint32_t pre_int=0;
	uint32_t repeat_found=0;
	uint32_t repeat_total=0;
	uint32_t table_miss = 0;
	uint32_t table_found = 0;
	uint32_t cache_found = 0;
	uint32_t buffer_found = 0;
	uint32_t patterns_total = 0;
	uint32_t patterns_freed = 0;
	uint32_t repeat_sub=0;
	uint32_t largest_id_size=0;
#endif
	uint16_t member_length=0;
	uint16_t search_offset = 0;
	uint32_t saved=0;
	uint32_t prefetch_a[128];
	uint32_t prefetch_b[8];
	uint32_t pre_count=0;
	uint32_t pre_ret=0;
	unsigned char first_char=0;
	unsigned char last_char=0;
	unsigned char sec_char=0;
	unsigned char a_char=0;
	unsigned char b_char=0;
	unsigned char c_char=0;
	uint32_t id = 0;
	char *index;
	uint32_t sizes[256][256];
	struct buffer_structure *buffer[256];
	struct common_structure *common[256];
	uint32_t buf_members[256];
	uint32_t buf_count[256];
	uint32_t com_count[256];
	uint32_t *unique_id[256];
	uint32_t *common_id[256];
	uint32_t *buffer_id[256];
	uint32_t tbl_mems[256];
	uint32_t *buffer_open[256];
	uint32_t *common_open[256];
	struct translation_structure *temp_struct_array;
	uint32_t cache_sizes[2];
	uint16_t b_members=0;
	uint16_t c_members=0;
	uint32_t tbl_count = 1;
	size_t tid = *(int *)set;
	uint32_t thread = table[tid].search_num;
	struct timeval starttime,endtime;
	double time1=0;

	while(tbl_count <= search_members[thread][0]){
		gettimeofday(&starttime, NULL);
		num1=search_members[thread][tbl_count];
		pre_int=0;
		repeat_found=0;
		repeat_total=0;
		table_miss = 0;
		table_found = 0;
		cache_found = 0;
		buffer_found = 0;
		patterns_total = 0;
		patterns_freed = 0;
		repeat_sub=0;
		pre_ret=0;
		pre_count=0;
		saved=0;
		member_length = table[num1].len-1;
		stop_byte = file_bytes-table[num1].len-128;
		if(member_length>5){
			index = malloc(131072);
		}else{
			index = malloc(65536);
		}
		num3 = (sizeof(struct buffer_structure)+sizeof(uint32_t)+sizeof(uint32_t)+member_length+1)<<8;
		num4 = 655360/num3; //truncated to int
		num3 = (sizeof(struct common_structure)+sizeof(uint32_t)+member_length+1)<<8;
		num5 = 262144/num3; //truncated to int
		if(num4 < 8){
			num4=8;
		}
		while(num4 % 8 != 0){
			num4++;
		}
		while(num5 % 4 != 0){
			num5++;
		}
		b_members = num4;
		c_members = num5;
		for(num3=0; num3<256; num3++){
			for(num4=0; num4<256; num4++){
				sizes[num3][num4]=1;
			}
			buffer[num3] = malloc(b_members*sizeof(struct buffer_structure));
			common[num3] = malloc(c_members*sizeof(struct common_structure));
			buffer_id[num3] = malloc(b_members*sizeof(uint32_t));
			common_id[num3] = malloc(c_members*sizeof(uint32_t));
			buffer_open[num3] = malloc(b_members*sizeof(uint32_t));
			common_open[num3] = malloc(c_members*sizeof(uint32_t));
			for(num4=0; num4<b_members; num4++){
				buffer[num3][num4].string = malloc(table[num1].len*sizeof(char));
			}
			for(num4=0; num4<c_members; num4++){
				common[num3][num4].string = malloc(table[num1].len*sizeof(char));
			}
		}
		cache_sizes[0] = ((b_members*( sizeof(struct buffer_structure)+sizeof(uint32_t)+sizeof(uint32_t)+(table[num1].len*sizeof(char)) ))<<8);
		cache_sizes[1] = ((c_members*( sizeof(struct common_structure)+sizeof(uint32_t)+(table[num1].len*sizeof(char)) ))<<8);
		//first_char loop
		for(num7=255; num7>-1; num7--){
			num2=0;
			reset_index(member_length, index);
			for(num3=0; num3<256; num3++){
				buf_members[num3] = 0;
				buf_count[num3] = 0;
				com_count[num3]=0;
				tbl_mems[num3]=0;
				for(num4=0; num4<b_members; num4++){
					buffer_open[num3][num4]=1;
					buffer[num3][num4].times=0;
				}
				for(num4=0; num4<c_members; num4++){
					common[num3][num4].times=0;
					common[num3][num4].sub=0;
					common_open[num3][num4]=1;
				}
				unique_id[num3]=malloc(sizeof(uint32_t)<<1);
				unique_id[num3][0]=0;
			}
			for(num3=0; num3<8; num3++){
				prefetch_b[num3] = num7;
			}
			first_char = num7;
			repeat_found=0;
			//entire file scan loop
			while(num2<stop_byte){
				//fill a page 16bytes at a time
				for(num3=0; num3<121; num3+=8){
					prefetch_a[num3] = (unsigned char)mem_file[num2+num3];
					prefetch_a[num3+1] = (unsigned char)mem_file[num2+num3+1];
					prefetch_a[num3+2] = (unsigned char)mem_file[num2+num3+2];
					prefetch_a[num3+3] = (unsigned char)mem_file[num2+num3+3];
					prefetch_a[num3+4] = (unsigned char)mem_file[num2+num3+4];
					prefetch_a[num3+5] = (unsigned char)mem_file[num2+num3+5];
					prefetch_a[num3+6] = (unsigned char)mem_file[num2+num3+6];
					prefetch_a[num3+7] = (unsigned char)mem_file[num2+num3+7];
				}
				//parse page for num7 matches
				for(num3=0; num3<121; num3+=8){
					if(prefetch_a[num3] == prefetch_b[0]){
						prefetch_a[num3]=-1;
					}
					if(prefetch_a[num3+1] == prefetch_b[1]){
						prefetch_a[num3+1]=-1;
					}
					if(prefetch_a[num3+2] == prefetch_b[2]){
						prefetch_a[num3+2]=-1;
					}
					if(prefetch_a[num3+3] == prefetch_b[3]){
						prefetch_a[num3+3]=-1;
					}
					if(prefetch_a[num3+4] == prefetch_b[4]){
						prefetch_a[num3+4]=-1;
					}
					if(prefetch_a[num3+5] == prefetch_b[5]){
						prefetch_a[num3+5]=-1;
					}
					if(prefetch_a[num3+6] == prefetch_b[6]){
						prefetch_a[num3+6]=-1;
					}
					if(prefetch_a[num3+7] == prefetch_b[7]){
						prefetch_a[num3+7]=-1;
					}
				}
				//run the matches
				pre_int=num2;
				pre_count=0;
				while(pre_count<125){
					if(prefetch_a[pre_count] == -1){
						num2=pre_int+pre_count;
						pre_ret=1;
						goto proceed;
					}
					pre_a:
					pre_count++;
					if(prefetch_a[pre_count] == -1){
						num2=pre_int+pre_count;
						pre_ret=2;
						goto proceed;
					}
					pre_b:
					pre_count++;
					if(prefetch_a[pre_count] == -1){
						num2=pre_int+pre_count;
						pre_ret=3;
						goto proceed;
					}
					pre_c:
					pre_count++;
					if(prefetch_a[pre_count] == -1){
						num2=pre_int+pre_count;
						pre_ret=4;
						goto proceed;
					}
					pre_d:
					pre_count++;
				}
				goto found;
				proceed:
				sec_char = (unsigned char)mem_file[num2+1];
				last_char = (unsigned char)mem_file[num2+member_length];
				if(member_length>5){
					a_char = (unsigned char)mem_file[num2+member_length-1];
					b_char = (unsigned char)mem_file[num2+member_length-2];
					search_offset = member_length-3;
					c_char = (unsigned char)mem_file[num2+search_offset];
					id = c_char*16777216 + b_char*65536 + a_char*256 + last_char;
				}else{
					search_offset = member_length-1;

					a_char = (unsigned char)mem_file[num2+search_offset];
					id = first_char*16777216 + sec_char*65536 + a_char*256 + last_char;
				}
				goto repeat_search;
				buffer_search:
				num3=0;
				//search cached xMB worth of most recent found patterns
				while(num3<b_members){
					if(!buffer_open[last_char][num3]){
						if(id != buffer_id[last_char][num3]){
							goto r_next;
						}
						for( num4=1; num4 < search_offset; num4++){
							if( mem_file[num2+num4] != buffer[last_char][num3].string[num4] ){
								goto r_next;
							}
						}
						buffer_found++;
						buffer[last_char][num3].times++;
						if(pre_ret == 1){
							goto pre_a;
						}
						if(pre_ret == 2){
							goto pre_b;
						}
						if(pre_ret == 3){
							goto pre_c;
						}
						goto pre_d;
					}
					r_next:
					num3++;
				}
				//search cached xMB worth of most often found patterns
				num3=0;
				while(num3<c_members){
					if(!common_open[last_char][num3]){
						if(id != common_id[last_char][num3]){
							goto o_next;
						}
						for( num4=1; num4 < search_offset; num4++){
							if( mem_file[num2+num4] != common[last_char][num3].string[num4] ){
								goto o_next;
							}
						}
						cache_found++;
						common[last_char][num3].times++;
						if(pre_ret == 1){
							goto pre_a;
						}
						if(pre_ret == 2){
							goto pre_b;
						}
						if(pre_ret == 3){
							goto pre_c;
						}
						goto pre_d;
					}
					o_next:
					num3++;
				}
				//table search
				num3 = unique_id[last_char][0];
				while(num3>0){
					if(id == unique_id[last_char][num3]){
						saved=table[num1].member[first_char][last_char][num3].series;
						if( Compare(saved+1, num2+1, search_offset) ){
							goto next5;
						}
						goto id_match;
					}
					next5:
					num3--;
				}
				table_miss++;
				goto no_search;
				id_match:
				table[num1].member[first_char][last_char][num3].times++;
				for(num4=com_count[last_char]; num4<c_members; num4++){
					if(table[num1].member[first_char][last_char][num3].times>common[last_char][num4].times){
						//contribute old results to table before replacing
						if(!common_open[last_char][num4]){
							num6 = common[last_char][num4].sub;
							table[num1].member[first_char][last_char][num6].times=common[last_char][num4].times;
						}
						//write new results to common index
						common_open[last_char][num4]=0;
						common[last_char][num4].times=table[num1].member[first_char][last_char][num3].times;
						common_id[last_char][num4]=id;
						common[last_char][num4].sub=num3;
						for(num6=0; num6<member_length+1; num6++){
							common[last_char][num4].string[num6]=mem_file[table[num1].member[first_char][last_char][num3].series+num6];
						}
						if(com_count[last_char] == c_members-1){
							com_count[last_char] = 0;
						}else{
							com_count[last_char]++;
						}
						table_found++;
						if(pre_ret == 1){
							goto pre_a;
						}
						if(pre_ret == 2){
							goto pre_b;
						}
						if(pre_ret == 3){
							goto pre_c;
						}
						goto pre_d;
					}
				}
				table_found++;
				if(pre_ret == 1){
					goto pre_a;
				}
				if(pre_ret == 2){
					goto pre_b;
				}
				if(pre_ret == 3){
					goto pre_c;
				}
				goto pre_d;
				repeat_search:
				;
				if(first_char != (unsigned char)mem_file[num2+member_length]){
					if(member_length>5){
						if(!index[(sec_char<<8)+last_char])
							goto no_search;
						if(!index[65536+(a_char<<8)+b_char])
							goto no_search;
						goto buffer_search;
					}
					if(!index[(sec_char<<8)+last_char])
						goto no_search;
					goto buffer_search;
				}
				//check if it's a repeat
				for(num3=1; num3<member_length; num3++){
					if(first_char != (unsigned char)mem_file[num2+num3]){
						if(member_length>5){
							if(!index[(sec_char<<8)+last_char])
								goto no_search;
							if(!index[65536+(a_char<<8)+b_char])
								goto no_search;
							goto buffer_search;
						}
						if(!index[(sec_char<<8)+last_char])
							goto no_search;
						goto buffer_search;
					}
				}
				repeat_found++;
				if(repeat_found == 1)
					repeat_sub = num2;
				if(pre_ret == 1){
					goto pre_a;
				}
				if(pre_ret == 2){
					goto pre_b;
				}
				if(pre_ret == 3){
					goto pre_c;
				}
				goto pre_d;
				no_search:
				;
				if(buf_members[last_char]<b_members){
					for(num3=0; num3<member_length+1; num3++){
						buffer[last_char][ buf_count[last_char] ].string[num3]=mem_file[num2+num3];
					}
					buffer[last_char][ buf_count[last_char] ].series=num2;
					buffer_id[last_char][ buf_count[last_char] ] = id;
					buffer[last_char][ buf_count[last_char] ].times = 1;
					buffer_open[last_char][buf_count[last_char]]=0;

					if(member_length>5){
						index[(sec_char<<8)+last_char]=1;
						index[65536+(a_char<<8)+b_char]=1;
					}else{
						index[(sec_char<<8)+last_char]=1;
					}
					buf_count[last_char]++;
					buf_members[last_char]++;
				}else{
					//take 4 oldest out of buffer and place into table
					//dealing with sets of 4
					if(buf_count[last_char] == b_members){
						buf_count[last_char] = 0;
					}

					for(num3=buf_count[last_char]; num3<buf_count[last_char]+4; num3++){
						num5 = table[num1].member[first_char][last_char][0].times;
						if(sizes[first_char][last_char] == num5){
							table[num1].member[first_char][last_char]=realloc(table[num1].member[first_char][last_char],sizeof(struct translation_structure) * (sizes[first_char][last_char]+1+3) );
							unique_id[last_char]=realloc(unique_id[last_char], sizeof(uint32_t) * (sizes[first_char][last_char]+1+3) );
							sizes[first_char][last_char]+=3;
						}
						table[num1].member[first_char][last_char][num5+1].series = buffer[last_char][num3].series;
						table[num1].member[first_char][last_char][num5+1].times = buffer[last_char][num3].times;
						table[num1].member[first_char][last_char][0].times++;
						unique_id[last_char][ num5+1 ] = buffer_id[last_char][num3];
						unique_id[last_char][0] = table[num1].member[first_char][last_char][0].times;
						buffer_open[last_char][num3]=1;
						//add to common index if popular...
						for(num4=com_count[last_char]; num4<c_members; num4++){
							if(buffer[last_char][num3].times > common[last_char][num4].times){
								//contribute old results to table before replacing
								if(!common_open[last_char][num4]){
									num6 = common[last_char][num4].sub;
									table[num1].member[first_char][last_char][num6].times=common[last_char][num4].times;
								}
								//write new results to common index
								common_open[last_char][num4]=0;
								common[last_char][num4].times=buffer[last_char][num3].times;
								common_id[last_char][num4]=buffer_id[last_char][num3];
								common[last_char][num4].sub=num5+1;
								for(num6=0; num6<member_length+1; num6++){
									common[last_char][num4].string[num6]=buffer[last_char][num3].string[num6];
								}
								if(com_count[last_char] == c_members-1){
									com_count[last_char] = 0;
								}else{
									com_count[last_char]++;
								}
							}
						}
					}
					//put current pattern into buffer
					for(num3=0; num3<member_length+1; num3++){
						buffer[last_char][buf_count[last_char]].string[num3]=mem_file[num2+num3];
					}
					buffer[last_char][buf_count[last_char]].series=num2;
					buffer_id[last_char][buf_count[last_char]] = id;
					buffer[last_char][buf_count[last_char]].times = 1;
					buffer_open[last_char][buf_count[last_char]]=0;

					if(member_length>5){
						index[(sec_char<<8)+last_char]=1;
						index[65536+(a_char<<8)+b_char]=1;
					}else{
						index[(sec_char<<8)+last_char]=1;
					}
					buf_count[last_char]++;
					buf_members[last_char]-=3;
				}
				found:
				num2=pre_int+pre_count;
			}
			if(repeat_found != 0){
				repeat_total += repeat_found;
				num5 = table[num1].member[first_char][first_char][0].times;
				if(sizes[first_char][first_char] == num5){
					table[num1].member[first_char][first_char]=realloc(table[num1].member[first_char][first_char], sizeof(struct translation_structure) * (sizes[first_char][first_char]+1+3) );
					sizes[first_char][first_char]+=3;
				}
				table[num1].member[first_char][first_char][0].times++;
				tbl_mems[last_char]++;
				table[num1].member[first_char][first_char][num5+1].times=repeat_found;
				table[num1].member[first_char][first_char][num5+1].series=repeat_sub;
				if(repeat_found > highest_times){
					highest_times=repeat_found;
				}
			}
			num4 = largest_id_size;
			for(num3=0; num3<256; num3++){
				largest_id_size += sizes[first_char][num3]+1;
				free(unique_id[num3]);
			}
			if(num4>largest_id_size)
				largest_id_size=num4;
			//save all needed index info to tables before reset
			for(num3=0; num3<256; num3++){
				for(num6=0; num6<b_members; num6++){
					if(!buffer_open[num3][num6]){
						num5 = table[num1].member[first_char][num3][0].times;
						if(sizes[first_char][num3] == num5){
							table[num1].member[first_char][num3]=realloc(table[num1].member[first_char][num3], sizeof(struct translation_structure) * (sizes[first_char][num3]+1+3) );
							sizes[first_char][num3]+=3;
						}
						table[num1].member[first_char][num3][0].times++;
						tbl_mems[last_char]++;
						table[num1].member[first_char][num3][num5+1].series=buffer[num3][num6].series;
						table[num1].member[first_char][num3][num5+1].times=buffer[num3][num6].times;
					}
				}
				for(num4=0; num4<c_members; num4++){
					if(!common_open[num3][num4]){
						num6 = common[num3][num4].sub;
						table[num1].member[first_char][num3][num6].times=common[num3][num4].times;
					}
				}
			}
			//free all strings that don't meet min frequency to save memory later
			for(num3=0; num3<256; num3++){
				num5=table[num1].member[first_char][num3][0].times+1;
				num6=1;
				temp_struct_array = malloc(sizeof(struct translation_structure)*num5 );
				temp_struct_array[0].times = table[num1].member[first_char][num3][0].times;
				temp_struct_array[0].series=0;
				for(num4=1; num4<num5; num4++){
					if(table[num1].member[first_char][num3][num4].times>frequency-1){
						if(table[num1].member[first_char][num3][num4].times>highest_times){
							highest_times = table[num1].member[first_char][num3][num4].times;
						}
						temp_struct_array[num6].series=table[num1].member[first_char][num3][num4].series;
						temp_struct_array[num6].times=table[num1].member[first_char][num3][num4].times;
						num6++;
					}else{
						temp_struct_array[0].times--;
					}
				}
				patterns_freed+=sizes[first_char][num3]+1-num6;
				free(table[num1].member[first_char][num3]);
				table[num1].member[first_char][num3]=malloc(sizeof(struct translation_structure)*num6 );
				for(num4=0; num4<num6; num4++){
					table[num1].member[first_char][num3][num4].series=temp_struct_array[num4].series;
					table[num1].member[first_char][num3][num4].times=temp_struct_array[num4].times;
				}
			}
		}

		table[num1].highest_times=highest_times;
		if(Verbose){
			progress_func(0, member_length+1);
			printf("\tbuffer/cache sizes\n");
			printf("\t\trecent: %u(%.02lfKB)", b_members, (double)cache_sizes[0]/1024);
			printf(" common: %u(%.02lfKB)", c_members, (double)cache_sizes[1]/1024);
			if(member_length>5){
				printf(" index: 128KB");
			}else{
				printf(" index: 64KB");
			}
			printf(" unique_id: %.02lfKB\n", (double)largest_id_size*sizeof(uint32_t)/1024 );
			largest_id_size=0;
			patterns_total = table_found + buffer_found + cache_found + table_miss + repeat_total;
			if(patterns_total == 0){
				printf("WARING: Exiting to prevent floating-point exception!\n");
				printf("\tNo patterns found.\n");
				exit(1);
			}
			printf("\ttotal patterns found: %u\n", patterns_total-table_miss);
			printf("\ttable_miss: %u table_found: %u recent_found: %u common_found: %u repeat_found: %u\n", table_miss, table_found, buffer_found, cache_found, repeat_total);
			printf("\ttable_miss: %.02lf%% table_found: %.02lf%% recent_found: %.02lf%% common_found: %.02lf%% repeat_found: %.02lf%%\n", (double)table_miss/patterns_total*100, (double)table_found/patterns_total*100, (double)buffer_found/patterns_total*100, (double)cache_found/patterns_total*100, (double)repeat_total/patterns_total*100 );
			printf("\tfreed: %u (%.04lfMB) low freq pattern structures\n", patterns_freed, (double) (patterns_freed*sizeof(struct translation_structure))/1024/1024 );
		}
		for(num3=0; num3<256; num3++){
			free(buffer[num3]);
			free(common[num3]);
			free(common_id[num3]);
			free(buffer_id[num3]);
			free(buffer_open[num3]);
			free(common_open[num3]);
		}
		free(index);
		tbl_count++;
		gettimeofday(&endtime, NULL);
		time1=((double)(endtime.tv_sec*1000000-starttime.tv_sec*1000000+endtime.tv_usec-starttime.tv_usec))/1000000;
		if(Verbose)
			printf("\tHighest times: %u Pattern scan Rate: %.02lfKB/s\n", highest_times, (double)(file_bytes>>10)/time1);
	}
	pthread_exit(0);
}

void Patterns_PT(int verbose, int thread_count, double *results){
	//FILE *output;
	Verbose = verbose;
	int input;
	struct stat buffer;
	int status;
	int pat_size = 4;
	int num1, num2, num3, num4, num5;
	int max_searchs=0;
	int max_pat = 16;
	int half_file;
	frequency=4;
	//int thread_count;
	struct timeval starttime,endtime;
	double time1;
	//if(argc != 6){
	//	usage();
	//}
	//thread_count = atoi(argv[5]);
	//pat_size = atoi(argv[3]);
	//max_pat = atoi(argv[4]);
	input = open("patterns.data", O_RDONLY);
	//input = open(argv[2], O_RDONLY);
	if(input == -1){
		fprintf( stderr, "Error opening ./patterns.data\n");
		exit( 1 );
	}
	status = fstat(input, &buffer);
	file_bytes=buffer.st_size;
	if(file_bytes < max_pat+128){
		printf("Error: file is too small!\n");
		exit(1);
	}
#ifndef BIG_FILE
	if(file_bytes > 4294967296-max_pat){
		printf("Error: this file exceeds 4GB, recompile with -D BIG_FILE\n");
		exit(1);
	}
#endif
	mem_file=(char *)mmap(0, file_bytes, PROT_READ, MAP_SHARED, input, 0);
	if (mem_file == MAP_FAILED) {
		close(input);
		perror("Error mmapping the file");
		exit(EXIT_FAILURE);
	}

	//if( ( output = fopen( argv[1], "w" ) ) == NULL ) {
	//	fprintf( stderr, "Error opening %s\n", argv[1] );
	//	exit( 1 );
	//}
	if(Verbose){
		printf("input: ./patterns.data\n");
		//printf("output: %s\n", argv[1]);
		printf("pat_size: %d\n", pat_size);
		printf("bytes: %d\n", file_bytes);
	}
	if(max_pat > file_bytes/2){
		max_pat = floor(file_bytes/2);
		printf("max_pat: %d (reduced to limit!)\n", max_pat);
	}else{
		if(Verbose)
			printf("max_pat: %d\n", max_pat);
	}
	half_file = sizeof(unsigned char)*ceil(file_bytes/2);
	for(num1=pat_size; num1<max_pat; num1++){
		max_searchs++;
	}
	max_searchs++;
	if(Verbose){
		printf("max_searchs: %d\n", max_searchs);
		printf("frequency: %d\n", frequency);
	}
	if(thread_count > max_searchs){
		thread_count = max_searchs;
		printf("threads: %d(reduced!)\n", thread_count);
	}else{
		if(Verbose)
			printf("threads: %d\n", thread_count);
	}
	thread_id = (int *)malloc(sizeof(int) * thread_count);
	pthread_t *threads = (pthread_t *)malloc(sizeof(pthread_t) * thread_count);
	table=(struct pat_length *)malloc( sizeof(struct pat_length) * max_searchs);
	for(num1=0; num1<max_searchs; num1++){
		table[num1].member=malloc( 256*sizeof(void **) );
		for(num2=0; num2<256 ; num2++){
			table[num1].member[num2]=malloc( 256*sizeof(void *) );
			for(num3=0; num3<256; num3++){
				table[num1].member[num2][num3]=malloc( 2*sizeof(struct translation_structure) );
				table[num1].member[num2][num3][0].times=0;
			}
		}
	}
	search_members = (uint32_t **)malloc(sizeof(uint32_t *) * thread_count);
	num2 = ceil(max_searchs/thread_count);
	for(num1=0; num1<thread_count; num1++){
		thread_id[num1] = (int)num1;
		//fix later
		search_members[num1] = (uint32_t *)malloc(sizeof(uint32_t) * (num2+2));
	}
	for(num1=0; num1<thread_count; num1++)
		search_members[num1][0] = 0;

	num2=0;
	for(num1=0; num1<max_searchs; num1++){
		table[num1].len = pat_size+num1;
		table[num1].search_num = num1;
		table[num1].highest_times = 0;
		search_members[num2][0]++;
		num3 = search_members[num2][0];
		search_members[num2][num3] = num1;
		if(num2 == thread_count-1){
			num2=0;
		}else{
			num2++;
		}

	}
//////////////////////////////////
	if(Verbose)
		progress_func(max_searchs, 0);
	gettimeofday(&starttime, NULL);
	for(num1=0; num1<thread_count; num1++){
		if(pthread_create(&threads[num1], NULL, match_search, (void*)&thread_id[num1]) != 0)
			perror("pthread_create"), exit(1);
	}
	for(num1=0; num1<thread_count; num1++){
		if (pthread_join(threads[num1], NULL) != 0)
			perror("pthread_join"),exit(1);
	}
	gettimeofday(&endtime, NULL);
	time1=((double)(endtime.tv_sec*1000000-starttime.tv_sec*1000000+endtime.tv_usec-starttime.tv_usec))/1000000;
	if(verbose){
		printf("time: %.02lfs\n", time1);
		//printf("generating output file...\n");
	}
	//gettimeofday(&starttime, NULL);
	/*
	//parse structures and write to output
	int index1=0;
	int index2=0;
	int max_print=0;
	int num6;
	struct sort_structure *sort_index;
	sort_index = malloc(max_searchs*sizeof(struct sort_structure));
	//initialize sort_index to all 0's
	for(num5=0; num5<max_searchs; num5++){
		sort_index[num5].highest=0;
		for(num6=0; num6<256; num6++){
			for(index1=0; index1<256; index1++){
				sort_index[num5].sub_highest[num6][index1]=0;
			}
		}
	}
	//search tables for the one containing the highest count
	num2=0;
	for(num1=0; num1<max_searchs; num1++){
		//find highest member count
		sort_index[num1].highest=table[num1].highest_times;
		if(sort_index[num1].highest > num2){
			num2=sort_index[num1].highest;
		}
	}
	//search tables for highest in all sets and save to sort_index
	for(num1=max_searchs-1; num1>-1; num1--){
		for(index1=0; index1<256; index1++){
			for(index2=0; index2<256; index2++){
				num5=table[num1].member[index1][index2][0].times+1;
				for(num4=1; num4<num5; num4++){
					if(table[num1].member[index1][index2][num4].times > sort_index[num1].sub_highest[index1][index2]){
						sort_index[num1].sub_highest[index1][index2]=table[num1].member[index1][index2][num4].times;
					}
				}
			}
		}
	}

	while(num2>=frequency){
		for(num1=max_searchs-1; num1>-1; num1--){
			if(sort_index[num1].highest == num2){
				sort_index[num1].highest = 0;
				for(index1=0; index1<256; index1++){
					for(index2=0; index2<256; index2++){
						if(sort_index[num1].sub_highest[index1][index2] == num2){
							sort_index[num1].sub_highest[index1][index2]=0;
							num5=table[num1].member[index1][index2][0].times+1;
							for(num4=1; num4<num5; num4++){
								if(table[num1].member[index1][index2][num4].times == num2){
									fprintf(output, "%d %d", table[num1].len, table[num1].member[index1][index2][num4].times);
									for(num3=0; num3<table[num1].len; num3++){
										fprintf(output, " %d", mem_file[table[num1].member[index1][index2][num4].series+num3]);
									}
									fprintf(output, "\n");
									max_print++;
									if(max_print == 40000){
										goto complete;
									}
								}else if(table[num1].member[index1][index2][num4].times < num2){
									if(table[num1].member[index1][index2][num4].times > sort_index[num1].sub_highest[index1][index2]){
										sort_index[num1].sub_highest[index1][index2] = table[num1].member[index1][index2][num4].times;
									}
								}
							}
						}
						if(sort_index[num1].sub_highest[index1][index2] > sort_index[num1].highest){
							sort_index[num1].highest=sort_index[num1].sub_highest[index1][index2];
						}
					}
				}
			}
		}
		num2=0;
		for(num6=0; num6<max_searchs; num6++){
			//find highest member count
			if(sort_index[num6].highest > num2){
				num2=sort_index[num6].highest;
			}
		}
	}
	complete:
	;
	gettimeofday(&endtime, NULL);
	time1=((double)(endtime.tv_sec*1000000-starttime.tv_sec*1000000+endtime.tv_usec-starttime.tv_usec))/1000000;
	printf("time: %.02lfs\n", time1);
	*/

	//system("ps -C lsbench2 -o rss=");
	if(munmap(mem_file, file_bytes) == -1) {
		perror("Error un-mmapping the file");
	}
	for(num1=0; num1<thread_count; num1++){
		free(search_members[num1]);
	}
	free(search_members);
	free(thread_id);
	close(input);
	//fclose(output);
	if(Verbose)
		printf("done.\n");
	results[1] = (double)file_bytes/1024/time1;
	results[1] *= max_searchs;
	results[0] += results[1];

	//return 0;
}