kravets - us6704727 – appendices a and b (with page/line ... · kravets - us6704727 –...

Kravets - US6704727 – Appendices A and B (with page/line numbers for citation) – Page 1

1 2 8 9 11 12 13 14


function refreshAndScrollCache(prefix) {//force reload 1 SB.document.location.replace{'/d/sc/?prefix=' + escape{prefix) + '&source=generic') 2 } 3 4 function makeHref(term){ 5 return ( 6 ''+ term +'
\n') 8 } 9 10 function indexOfMinimum(array){ 11 var minimum = array[0] 12 var minindex = 0 13 for (var i in array) { 14 if(array[i]


list[mi] = cache[key][i] 1 listValues[mi] = values[key][i] 2 } 3 }//else 4 } //for all in cache[key] 5 //At this point list should contain 5 highest value words > prefix, 6 //just alpha sort and show it 7 list.sort() 8 if(list.length > 0){ 9 //give it a style 10 SB.document.write( 11 ' A {font-size:9pt; font-family: Helvetica, Arial}' + 12 'A:hover {color:red} A:active {color:red}\n') 13 for(var k in list) 14 SB.document.write(makeHref(list[k])) 15 SB.document.write('') 16 SB.document.close() 17 }else 18 showDefault() 19 } 20 //--> 21


package com.go2.search.predictive; 1 2 import java.util.*; 3 import java.net.*; 4 import java.io.*; 5 6 /** 7 * This class implements the in-memory Predictive Search cache 8 * At boot time a file with cache entries and their corresponding values 9 * is loaded and processed by this Class and thereafter a service 10 * is made available on a UDP port to quickly serve client requests 11 * @author Alex Kravets 12 */ 13 public class PredictiveServer { 14 15 /** 16 * Bind a UDP port and listen for requests 17 * @param args java.lang.String[] - Argumenst: /cache/file/path udpPortToBind 18 */ 19 public static void main(String args[]) { 20 //Simple usage check 21 if(args.length != 3) throw new IllegalArgumentException( 22 "Usage: java com.go2.search.predictive.Predictiveserver /path/to/cache perPrefixMax udpPortToBind"); 23 24 System.out.println("Starting Predictive Server ...\nTotal JVM Mem Usage Before cache load: " + 25 (Runtime.getRuntime().totalMemory())/1000000 + " Megabytes: "); 26 try { 27 //Load our cache from a file and maintain some stats about it 28 long start = System.currentTimeMillis(); 29 loadCache(args[0]); 30 pruneCache(Integer.parseInt(args[1])); 31 buildMap(); 32 long end = System.currentTimeMillis(); 33 System.gc(); //Free up some RAM 34 35 printStats(start, end); 36 serviceLoop(Integer.parseInt(args[2])); 37 }catch(Exception e) { 38 System.out.println("Failed with: " + e); 39 e.printStackTrace(); 40 } 41 }//main 42 43 private static void loadCache(String cacheFile) throws IOException { 44 BufferedReader reader= new BufferedReader(new FileReader(cacheFile), 64*1024); 45 while(true) { 46 String line = reader.readLine(); 47 if(line == null) break; //Indicates we're done with this file 48 int valueSep = line.lastindexOf(VSEP); 49 //encode for safe transport in HTML/JavaScript 50 String phrase = line.substring(0, valueSep).trim(); 51 //Ignore phrases that are less than MIN_PHRASE_LENGTH chars 52 //greater than MAX_PHRASE_LENGTH or contain '\' 53 int len = phrase.length(); 54 if(len < MIN_PHRASE_LENGTH || len > MAX_PHRASE_LENGTH || phrase.indexOf('\\') != -1) 55 continue; 56 phraseLength += phrase.length(); 57 phraseCount++; 58 String prefix = phrase.substring(0, PREFIX_SIZE); 59


//encode for safe transport in HTML/JavaScript 1 phrase = cleanPhrase.(phrase); 2 String value = line.substring(valueSep + 1).trim(); 3 ArrayList prefixLines; 4 //Add a clean line into subBuffers for processing 5 if((prefixLines = (ArrayList)prefixMap.get(prefix)) != null) { 6 prefixLines.add(phrase + VSEP +value); //will need to disasemble again 7 }else{ //if we haven't seen this prefix before than create a holder for it 8 prefixLines = new ArrayList(); 9 prefixMap.put(prefix, prefixLines); 10 } 11 }//while more lines to read 12 reader.close(); 13 }//loadCache 14 15 private static void pruneCache(int keep) { 16

//For every prefix in the cache, leave highest "keep" in values 17 //and then resort alphabetically, remember and drop those prefixes 18 //that had less than MIN_PRERIX_ENTRIES values in them 19 ArrayList small = new ArrayList(); 20

//System.out.println("Prefixes with number of entries exceeding " + keep + ": "); 21 Iterator prefixIterator = prefixMap.keyset().iterator(); 22 while(prefixIterator.hasNext()){ 23 String prefix = (String)prefixIterator.next(); 24 ArrayList segment= (ArrayList)prefixMap.get(prefix); 25 int size= segment.size(); 26 prefixListSize += size; 27

//Sort entries in the descending order of values if necessary 28 if(size > keep) {//if prefix too big 29 //System.out.println("\'" + prefix + "' by " + (segment.size() - keep) + ". "); 30 Collections.sort(segment, new Comparator() { 31 public int compare(Object a, Object b) { 32 String aLine = (String)a; 33

String bLine = (String)b; 34 int aValue = Integer.parseInt(aLine.substring(aLine.lastindexOf(VSEP) + 1)); 35 int bValue = Integer.parseInt(bLine.substring(bLine.lastindexOf(VSEP) + 1)); 36 return bValue - aValue; //induce descending ordering 37

} 38 public boolean equals(Object another) {return super.equals(another); 39 }); 40 //Drop those of lower value 41 segment.subList(keep, segment.size()).clear(); 42 } 43 //Put in desc alpha order for further consumption by buildMap 44 Collections.sort(segment); 45 //Remember prefixes that are too short for subsequent removal from prefixMap 46 if(size < MIN_PREFIX_ENTRIES) 47 small.add(prefix); 48 }//for all prefixes 49 //Actually drop all those that are too short 50 System.out.println("Number of prefixes with fewer than "+MIN_PREFIX_ENTRIES+" entries: "+small.size()); 51 for(int i = 0; i < small.size(); i++) { 52 //System.out.println("Dropping" + small.get(i)); 53 prefixMap.remove(small.get(i)); 54 } 55 }//pruneCache 56 57 private static void buildMap() { 58 //Precompute HTML/Javascript for every prefix in the map 59 Iterator prefixIterator = prefixMap.keySet().iterator(); 60 while(prefixIterator.hasNext()){ 61 prefixcount++; 62 String prefix = (String)prefixIterator.next(); 63 //get segment and build Html for it 64 ArrayList segment= (ArrayList)prefixMap.get(prefixJ; 65 String html = HTML_HEADER; 66 //Make sure to add the prefix into it first !! 67


StringBuffer entries = new StringBuffer("var c = new Array(" + ESEP + prefix + ESEP); 1 StringBuffer values = new StringBuffer("var v = new Array(0"); 2 ListIterator lineIterator = segment.listIterator(); 3 while(lineIterator.hasNext()){ 4 String line = (String)lineIterator.next(); 5 int valueSep = line.lastIndexOf (VSEP); 6 //add Javascript String quotes 7 entries.append("," + ESEP + line.substring(0, valueSep) + ESEP); 8 values.append(',' + line.substring(valueSep + 1)); 9 }//all in prefix segment 10 entries.append(");\n"); 11 values.append(");\n"); 12 html += entries; 13 html += values; 14 //html += “for(var 1 in c) c[i] = unescape(c[i]);\n"; 15 html += "parent.setCache(" + ESEP + prefix + ESEP + ", c, v);\n"; 16 html += HTML_FOOTER; 17

prefixMap.put(prefix, html.getBytes()); 18 }//for all entries 19 }//buildMap 20 21 private static void serviceLoop(int port) throws SocketException { 22

//Bind our UDP port and set UDP in/out buffers high to improve net I/O 23 DatagramSocket socket= new DatagramSocket(port); 24 socket.setSendBufferSize(64*1024); 25 socket.setReceiveBufferSize(64*1024); 26 //Enter the eternal loop 27 while(true) ( 28

try { 29 DatagramPacket in= new DatagramPacket(inBuffer, inBuffer.length); 30 socket.receive(in); //block here until a request arrives 31 byte[] output= lookup(new String(inBuffer, 0, in.getLength())); 32 socket.send(new DatagramPacket(output, output.length, in.getAddress(), in.getPort())); 33 //System.out.println("Recieved: " + packet.getData().length+ "bytes"); 34

}catch(IOException ioe) { }//Indicates read failure, ignore and try again 35 }//eternal while 36

}//serviceLoop 37 38 private static String cleanPhrase(String dirty){ 39

return URLEncoder.encode(dirty).replace('+',' '); 40 /* 41 int apoindex = -1; 42 StringBuffer sb =new StringBuffer(dirty); 43 while((apoindex = dirty.indexOf(charToEscape, apoindex + 1)) != -1) 44 sb.insert(apoindex, '\\'); 45 return sb.toString(); 46 */ 47

}//cleanPhrase 48 49 private static byte[] lookup(String key) { 50 Object data= prefixMap.get(key); 51 return (data !=null) ? (byte[]) data : NOPREFIX_HTML; 52 }//lookup 53 54 private static void printStats(long start, long end) { 55

System.out.print("Total number of entries: " + phraseCount + "; "); 56 System.out.println("Average cache entry is: " + phraseLength/phraseCount +" characters"); 57 System.out.print("Total number of good prefixes: "+prefixcount + " "); 58 System.out.println("Average size of a good prefix list: " + (prefixListSize/prefixCount) + " entries"); 59 System.out.print("Time to load cache: " + (end - start)/1000.0 + " seconds."); 60 System.out.println(" Rate: "+ phraseCount/((end - start+ 1)/1000.0) +" per second"); 61 System.out.println("Total JVM Mem Usage After Cache Load: " + 62 (Runtime.getRuntime().totalMemory())/1000000 +" Megabytes: " + 63


Runtime.getRuntime().totalMemory()/phrasecount + "bytes per phrase"); 1 //System.out.println("Sample HTML: \n" + new String((byte[])prefixMap.get("afr"))); 2 }//printStats 3 4

//Constants 5 private static final int PREFIX_SIZE = 3; 6 private static final int MIN_PREFIX_ENTRIES = 3; 7 private static final int MIN_PHRASE_LENGTH = 4; 8 private static final int MAX_PHRASE_LENGTH = 30; 9 private static final String HTML_HEADER = "\n"; 10 private static final String HTML_FOOTER = ""; 11 private static final char ESEP '"'; 12 private static final char VSEP = ':'; 13 private static final byte[] 14 NOPREFIX_HTML=(HTML_HEADER+"parent.setCache(\"\", new Array(), new 15

Array());\n"+HTML_FOOTER).getBytes(); 16 17

//maps possible prefixes to byte-encoded HTML/JavaScript for them 18 private static final TreeMap prefixMap = new TreeMap(); 19 //preallocate a single receive buffer for all inbound requests 20 private static final byte[] inBuffer = new byte[16]; 21 22 //statistics related stuff 23 private static int phrasecount; 24 private static int phraseLength; 25 private static int prefixCount; 26 private static int prefixListSize; 27

}//class PredictiveServer 28

kravets - us6704727 – appendices a and b (with page/line ... · kravets - us6704727 –...

Documents