Repositories » TXT0 Read More
Clone URL:  
Pushed to one repository · View In Graph Contained in tip

Fixed byte-length computation for .t file, added sum-of-squares of a document vector as a third resource.

Changeset 316c0109c1e6

Parent 7aa7a5f55649

by Rup Palchowdhury

Changes to 7 files · Browse files at 316c0109c1e6 Showing diff from parent 7aa7a5f55649 Diff from another changeset...

Change 1 of 6 Show Entire File parser.c Stacked
 
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
 
 
 
 
 
 
 
 
 
 
 
 
134
135
136
137
138
139
140
141
142
143
144
145
146
 
 
 
 
 
 
 
 
 
 
 
 
147
148
149
 
158
159
160
161
162
 
 
 
 
 
 
163
164
165
166
167
168
169
170
 
 
171
172
 
173
174
175
 
187
188
189
190
191
192
193
194
195
196
 
 
 
197
198
199
 
213
214
215
216
217
 
 
 
 
 
 
218
219
220
221
222
223
224
225
 
 
 
226
227
228
 
267
268
269
270
 
271
272
273
 
285
286
287
288
289
290
291
292
293
294
 
119
120
121
 
 
 
 
 
 
 
 
 
 
 
 
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
 
 
 
 
 
 
 
 
 
 
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
 
160
161
162
 
 
163
164
165
166
167
168
169
170
171
172
173
174
 
 
175
176
177
178
179
180
181
182
 
194
195
196
 
 
197
198
199
200
 
201
202
203
204
205
206
 
220
221
222
 
 
223
224
225
226
227
228
229
230
231
232
233
 
 
 
234
235
236
237
238
239
 
278
279
280
 
281
282
283
284
 
296
297
298
 
 
 
 
 
299
300
@@ -119,31 +119,33 @@
 TFile* parse(Parser *parser, FILE *fp)  {   Tokenizer *toktxt, *tokid; - Token *token; - Stack mem; - TFile *tfile; - TDoc *tdoc; - unsigned lowmem, b_read; - uint32_t crc, n_term, n_txt, n_uterm; - int n, txtsize, m, intact; - char *septxt = " ;,.:`'\"?!(){}[]<>~^&*_-+=#$%@|\\/"; - char *sepid = " <>"; - Hash *hpost; - Node *nppost, *nppost_, *npterm, node__; - Post *post, *post_; + Token *token; + Stack mem; + TFile *tfile; + TDoc *tdoc; + unsigned lowmem, b_read; + uint32_t crc, n_term, n_txt, n_uterm, sumsq; + int n, n_txtbuf, m, intact, n_header, n_subheader; + char *septxt = " ;,.:`'\"?!(){}[]<>~^&*_-+=#$%@|\\/"; + char *sepid = " <>"; + Hash *hpost; + Node *nppost, *nppost_, *npterm, node__; + Post *post, *post_;     __initcrc();   reset(&mem, 3); - toktxt = newtokenizer(septxt, LOWERCASE, &mem); - tokid = newtokenizer(sepid, KEEPCASE, &mem); - hpost = newhash(NHASH, cmppost, hashpost); - token = newtoken(NULL, KB); - lowmem = KB; - n_term = n_uterm = b_read = 0; - tfile = newTFile(TREC); - tdoc = newTDoc(TREC); - txtsize = TXTBUFSIZE; - intact = 1; + toktxt = newtokenizer(septxt, LOWERCASE, &mem); + tokid = newtokenizer(sepid, KEEPCASE, &mem); + hpost = newhash(NHASH, cmppost, hashpost); + token = newtoken(NULL, KB); + lowmem = KB; + n_txt = n_term = n_uterm = sumsq = b_read = 0; + tfile = newTFile(TREC); + tdoc = newTDoc(TREC); + n_txtbuf = TXTBUFSIZE; + n_header = sizeof(THeader); + n_subheader = sizeof(TSubHeader) - (MAX_RSRC - TREC_RSRC) * sizeof(uint32_t); + intact = 1;     while((n = gettoken(token, fp, toktxt)) > 0) {   @@ -158,18 +160,23 @@
  if (tdoc->h->n_id <= 0)   intact = 0;   - /* fill tdoc->txt and empty the hash */ - n_txt = 0; + /* compute sum(tf^2), fill tdoc->txt and empty + * the hash */ + n_txt = 0; + n_term = 0; + n_uterm = 0; + sumsq = 0;   for (int i = 0; i < hpost->n; i++) {   for (Node *np = hpost->tab[i]; np != NULL; np = np->next) {   post = (Post *)(np->data);   m = sprintf(&(tdoc->txt[n_txt]), "%s %u ",   post->id, post->tf);   n_txt += m; - if (txtsize - n_txt < lowmem) - tdoc->txt = erealloc(tdoc->txt, txtsize <<= 1); + if (n_txtbuf - n_txt < lowmem) + tdoc->txt = erealloc(tdoc->txt, n_txtbuf <<= 1);   n_uterm++;   n_term += post->tf; + sumsq += post->tf * post->tf;   }   }   freehash(hpost, freepost); @@ -187,13 +194,13 @@
  if (tdoc->h->n_rsrc[0] == -1)   tdoc->h->n_rsrc[0] = 0;   - n_term = 0; -   tdoc->h->n_rsrc[1] = sprintf(tdoc->rsrc[1], "%u", n_uterm);   if (tdoc->h->n_rsrc[1] == -1)   tdoc->h->n_rsrc[1] = 0;   - n_uterm = 0; + tdoc->h->n_rsrc[2] = sprintf(tdoc->rsrc[2], "%u", sumsq); + if (tdoc->h->n_rsrc[2] == -1) + tdoc->h->n_rsrc[2] = 0;     /* fill remaining parts of TDoc->TSubHeader */   crc = crc_wordwise(&crc_m, 0, NULL, 0); @@ -213,16 +220,20 @@
    /* update TFile->THeader */   tfile->h->n++; - tfile->h->b += TSUBHEADER_SIZE + tdoc->h->n_txt - + tdoc->h->n_id + tdoc->h->n_rsrc[0]; + tfile->h->b += n_subheader + + tdoc->h->n_txt + + tdoc->h->n_id + + tdoc->h->n_rsrc[0] + + tdoc->h->n_rsrc[1] + + tdoc->h->n_rsrc[2];   }   else   freeTDoc(tdoc, parser->type);     /* reset */ - intact = 1; - tdoc = newTDoc(TREC); - txtsize = TXTBUFSIZE; + intact = 1; + tdoc = newTDoc(TREC); + n_txtbuf = TXTBUFSIZE;     fprintf(stderr, "\rparsed: %d", tfile->h->n);   @@ -267,7 +278,7 @@
    b_read++;   - tfile->h->b += THEADER_SIZE; + tfile->h->b += n_header;     crc = crc_wordwise(&crc_m, 0, NULL, 0);   crc = crc_wordwise(&crc_m, crc, (unsigned char *)&tfile->h->ver, @@ -285,10 +296,5 @@
    tfile->h->crc = crc;   __deinitcrc(); - /* - fprintf(stderr, "%16s: %u\n", "bytes read", b_read); - fprintf(stderr, "%16s: %u\n", "TFile size", tfile->h->b); - fprintf(stderr, "%16s: %u\n", "#docs", tfile->h->n); - */   return tfile;  }
Change 1 of 1 Show Entire File test/​alice.mem Stacked
 
1
2
3
4
 
 
 
5
6
7
8
9
10
 
 
 
11
12
13
14
15
16
 
 
 
17
18
19
20
 
 
1
 
 
 
2
3
4
5
6
7
8
 
 
9
10
11
12
13
14
15
 
 
16
17
18
19
20
21
22
23
@@ -1,20 +1,23 @@
 MG CRC V T BYTES N R BO -41 3375973743 1 0 574 3 2 32 -CRC TXT ID RSRC0 RSRC1 -2702988819 181 2 2 2 +41 2797950090 1 0 622 3 3 32 +CRC TXT ID RSRC0 RSRC1 RSRC2 +2702988819 181 2 2 2 2  tdoc->id: A.  tdoc->txt: rude 1 open 1 curios 1 look 1 cut 1 alic 2 learn 1 hear 1 desk 1 veri 2 remark 1 great 1 hair 1 sever 1 wide 1 first 1 hatter 2 speech 1 write 1 ey 1 person 1 time 1 raven 1 make 1 .  tdoc->rsrc[0]: 27.  tdoc->rsrc[1]: 24. -CRC TXT ID RSRC0 RSRC1 -1863705286 187 2 2 2 +tdoc->rsrc[2]: 33. +CRC TXT ID RSRC0 RSRC1 RSRC2 +1863705286 187 2 2 2 2  tdoc->id: B.  tdoc->txt: adventur 1 issu 1 wrong 2 alic 1 1897 1 nevar 1 front 2 never 2 carrol 1 veri 1 propos 1 final 1 flat 1 note 1 himself 1 earli 1 spell 1 lewi 1 answer 1 produc 1 raven 1 though 1 revis 2 .  tdoc->rsrc[0]: 27.  tdoc->rsrc[1]: 23. -CRC TXT ID RSRC0 RSRC1 -711335375 126 2 2 2 +tdoc->rsrc[2]: 35. +CRC TXT ID RSRC0 RSRC1 RSRC2 +711335375 126 2 2 2 2  tdoc->id: C.  tdoc->txt: alic 1 both 1 dip 1 more 1 gave 1 1990 1 martin 1 gardner 1 flap 1 sent 1 answer 1 quill 1 reader 1 annot 1 possibl 1 slope 1 .  tdoc->rsrc[0]: 16.  tdoc->rsrc[1]: 16. +tdoc->rsrc[2]: 16.
 
 
 
1
2
3
4
 
 
 
5
6
7
8
9
10
 
 
 
11
12
13
14
15
16
 
 
 
17
18
19
20
21
22
 
 
 
23
24
25
26
 
 
1
 
 
 
2
3
4
5
6
7
8
 
 
9
10
11
12
13
14
15
 
 
16
17
18
19
20
21
22
 
 
23
24
25
26
27
28
29
30
@@ -1,26 +1,30 @@
 MG CRC V T BYTES N R BO -41 1553551552 1 0 312 4 2 32 -CRC TXT ID RSRC0 RSRC1 -944899761 63 2 1 1 +41 3101715852 1 0 368 4 3 32 +CRC TXT ID RSRC0 RSRC1 RSRC2 +944899761 63 2 1 1 1  tdoc->id: 1.  tdoc->txt: adventur 1 alic 1 desk 1 hatter 1 wonderland 1 write 1 raven 1 .  tdoc->rsrc[0]: 7.  tdoc->rsrc[1]: 7. -CRC TXT ID RSRC0 RSRC1 -497613859 48 2 1 1 +tdoc->rsrc[2]: 7. +CRC TXT ID RSRC0 RSRC1 RSRC2 +497613859 48 2 1 1 1  tdoc->id: 2.  tdoc->txt: puzzl 1 carrol 1 desk 1 lewi 1 answer 1 write 1 .  tdoc->rsrc[0]: 6.  tdoc->rsrc[1]: 6. -CRC TXT ID RSRC0 RSRC1 -85283453 68 2 1 1 +tdoc->rsrc[2]: 6. +CRC TXT ID RSRC0 RSRC1 RSRC2 +85283453 68 2 1 1 1  tdoc->id: 3.  tdoc->txt: pose 1 alic 1 puzzl 1 desk 1 hatter 1 wonderland 1 answer 1 write 1 .  tdoc->rsrc[0]: 8.  tdoc->rsrc[1]: 8. -CRC TXT ID RSRC0 RSRC1 -32326634 41 2 1 1 +tdoc->rsrc[2]: 8. +CRC TXT ID RSRC0 RSRC1 RSRC2 +32326634 41 2 1 1 1  tdoc->id: 4.  tdoc->txt: alic 1 more 1 martin 1 gardner 1 annot 1 .  tdoc->rsrc[0]: 5.  tdoc->rsrc[1]: 5. +tdoc->rsrc[2]: 5.
 
 
Change 1 of 8 Show Entire File tfile.c Stacked
 
19
20
21
22
 
23
24
25
 
44
45
46
47
 
48
49
50
 
71
72
73
74
 
75
76
77
 
216
217
218
219
 
220
221
222
 
252
253
254
255
 
256
257
258
 
337
338
339
340
 
341
342
343
 
380
381
382
383
 
384
385
386
 
400
401
402
403
 
404
405
406
 
19
20
21
 
22
23
24
25
 
44
45
46
 
47
48
49
50
 
71
72
73
 
74
75
76
77
 
216
217
218
 
219
220
221
222
 
252
253
254
 
255
256
257
258
 
337
338
339
 
340
341
342
343
 
380
381
382
 
383
384
385
386
 
400
401
402
 
403
404
405
406
@@ -19,7 +19,7 @@
  h->r = 0;   h->bo = 32;   if (type == TREC) - h->r = TREC_R; + h->r = TREC_RSRC;   return h;  }   @@ -44,7 +44,7 @@
  int r;   r = 0;   if (type == TREC) - r = TREC_R; + r = TREC_RSRC;   tdoc = (TDoc *)emalloc(sizeof(TDoc));   tdoc->h = _newTSubHeader(type);   tdoc->txt = (char *)emalloc(sizeof(char) * TXTBUFSIZE); @@ -71,7 +71,7 @@
  int r;   r = 0;   if (type == TREC) - r = TREC_R; + r = TREC_RSRC;   tdoc = (TDoc *)d;   if (tdoc == NULL)   return; @@ -216,7 +216,7 @@
    r = 0;   if (type == TREC) - r = TREC_R; + r = TREC_RSRC;     /* fill the TDoc->TSubHeader */   check = 0x0007; /* 0000 0000 0000 0111 */ @@ -252,7 +252,7 @@
  return NULL;   }   - /* read the TDoc->rsrc[i]->b */ + /* fill TDoc->rsrc[i] */   if (r > 0) {   v = 0;   check = 0x0000; @@ -337,7 +337,7 @@
  return NULL;   }   - /* read the TDoc->rsrc[i]->b */ + /* fill TDoc->rsrc[i] */   if (tfile->h->r > 0) {   v = 0;   check = 0x0000; @@ -380,7 +380,7 @@
  int r;   r = 0;   if (type == TREC) - r = TREC_R; + r = TREC_RSRC;     /* print header string */   printf("%-10s %-10s %-3s ", "CRC", "TXT", "ID"); @@ -400,7 +400,7 @@
  int r;   r = 0;   if (type == TREC) - r = TREC_R; + r = TREC_RSRC;     _printTSubHeader(tdoc->h, type);  
Change 1 of 1 Show Entire File tfile.h Stacked
 
1
2
3
4
5
6
 
 
 
 
7
8
9
 
 
 
 
 
 
 
1
2
3
4
5
6
7
@@ -1,9 +1,7 @@
-#define THEADER_SIZE 32 /* header size in bytes */ -#define TSUBHEADER_SIZE 12 /* sub-header size in bytes */ -#define TREC_R 2 /* number of resources for TDoc of type TREC */ -#define NRSRC 10 /* number of resources */ -#define TXTBUFSIZE 10240 /* 10KB */ -#define IDBUFSIZE 64 /* A UUID (if used) would be max 36 chars */ +#define TREC_RSRC 3 /* number of resources for TDoc of type TREC */ +#define MAX_RSRC 10 /* maximum number of resources */ +#define TXTBUFSIZE 10240 /* 10KB */ +#define IDBUFSIZE 64 /* A UUID (if used) would be max 36 chars */  #define RSRCBUFSIZE 32    typedef enum {TREC, WARC, TRECQUERY} E_TDocType;