-
Notifications
You must be signed in to change notification settings - Fork 4
/
utils.ts
91 lines (87 loc) · 3.09 KB
/
utils.ts
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
/**
* Tokenizes string into an array of tokens and filters out stopwords.
* @param name string to get tokenized
*/
function tokenize(name : string) : string[]{
let tokens : string[] = name.split(' ').join(',').split('.').join(',').split('(').join(',').split(')').join(',').split('-').join(',').split('_').join(',').split(',') // super super awful and nasty, but gets the job done.
tokens.forEach((value,index,array) => {array[index] = value.toLowerCase()})
tokens.forEach((value,index,array) => {array[index] = stemmer(value)})
return tokens
}
/**
* @param url location of the index of documents
*/
async function loadIndexFromURL(url : string) : Promise<Map<string,Object>>{
let response = await fetch(url)
let responsetext : string
if(response.ok){
responsetext = await response.text()
}else{
throw new Error(response.statusText);
}
let parsedResponse : Object[] = JSON.parse(responsetext)
let parsedIndex : Map<string,Object>
parsedIndex = new Map()
for(let object of parsedResponse){
let id = ""
let document = new Object()
for(let property of Object.keys(object)){
if(property === "id"){
id = object[property]
}else{
document[property] = object[property]
}
}
parsedIndex.set(id,document)
}
return parsedIndex
}
/**
* @returns a Promise that returns an index
*/
function loadInvertedIndexFromURL(url : string) : Promise<Map<string,Array<string>>>{
return fetch(url).then((response) => {
if(response.ok){
return response.text()
}
throw new Error("Couldn't fetch shard at URL " + url)
}).then((response) => {
let loadedIndex : Map<string,Array<string>> = new Map()
let lineNumber = 0
let lines = response.split("\n");
let version : number
lines.forEach((line) => {
if(lineNumber === 0){
if(parseInt(line) != 1 && parseInt(line) != 2){
throw "Error while parsing invinx: Invalid version, must be 1 or 2!"
}else{
version = parseInt(line)
}
lineNumber++
return
}
let cols = line.split(",")
let tokenname = decodeURIComponent(cols[0])
cols.shift()
if(version === 2){
cols = cols.map(function(value){
return value.replace("%2C",",")
})
}
loadedIndex.set(tokenname,cols)
lineNumber++
})
return(loadedIndex)
})
}
async function getDocumentForId(docid : string) : Promise<Object>{
docid = docid.replace("%2C",",")
await inxFetcher.fetchShard(inxFetcher.getIndexFor(docid))
if(inxFetcher.combinedIndex.get(docid) === undefined){
console.error("No document found for docid "+docid)
return {text: "no document found", id: docid}
}
let doc = inxFetcher.combinedIndex.get(docid)
doc["id"] = docid
return inxFetcher.combinedIndex.get(docid)
}