Fossil SCM

fossil-scm / src / cache.c
Blame History Raw 512 lines
1
/*
2
** Copyright (c) 2014 D. Richard Hipp
3
**
4
** This program is free software; you can redistribute it and/or
5
** modify it under the terms of the Simplified BSD License (also
6
** known as the "2-Clause License" or "FreeBSD License".)
7
**
8
** This program is distributed in the hope that it will be useful,
9
** but without any warranty; without even the implied warranty of
10
** merchantability or fitness for a particular purpose.
11
**
12
** Author contact information:
13
** [email protected]
14
**
15
*******************************************************************************
16
**
17
** This file implements a cache for expense operations such as
18
** /zip and /tarball.
19
*/
20
#include "config.h"
21
#include <sqlite3.h>
22
#include "cache.h"
23
24
/*
25
** Construct the name of the repository cache file
26
*/
27
static char *cacheName(void){
28
int i;
29
int n;
30
31
if( g.zRepositoryName==0 ) return 0;
32
n = (int)strlen(g.zRepositoryName);
33
for(i=n-1; i>=0; i--){
34
if( g.zRepositoryName[i]=='/' ){ i = n; break; }
35
if( g.zRepositoryName[i]=='.' ) break;
36
}
37
if( i<0 ) i = n;
38
return mprintf("%.*s.cache", i, g.zRepositoryName);
39
}
40
41
/*
42
** Attempt to open the cache database, if such a database exists.
43
** Make sure the cache table exists within that database.
44
*/
45
static sqlite3 *cacheOpen(int bForce){
46
char *zDbName;
47
sqlite3 *db = 0;
48
int rc;
49
i64 sz;
50
51
zDbName = cacheName();
52
if( zDbName==0 ) return 0;
53
if( bForce==0 ){
54
sz = file_size(zDbName, ExtFILE);
55
if( sz<=0 ){
56
fossil_free(zDbName);
57
return 0;
58
}
59
}
60
rc = sqlite3_open(zDbName, &db);
61
fossil_free(zDbName);
62
if( rc ){
63
sqlite3_close(db);
64
return 0;
65
}
66
sqlite3_busy_timeout(db, 5000);
67
if( sqlite3_table_column_metadata(db,0,"blob","key",0,0,0,0,0)!=SQLITE_OK ){
68
rc = sqlite3_exec(db,
69
"PRAGMA page_size=8192;"
70
"CREATE TABLE IF NOT EXISTS blob(id INTEGER PRIMARY KEY, data BLOB);"
71
"CREATE TABLE IF NOT EXISTS cache("
72
"key TEXT PRIMARY KEY," /* Key used to access the cache */
73
"id INT REFERENCES blob," /* The cache content */
74
"sz INT," /* Size of content in bytes */
75
"tm INT," /* Last access time (unix timestamp) */
76
"nref INT" /* Number of uses */
77
");"
78
"CREATE TRIGGER IF NOT EXISTS cacheDel AFTER DELETE ON cache BEGIN"
79
" DELETE FROM blob WHERE id=OLD.id;"
80
"END;",
81
0, 0, 0
82
);
83
if( rc!=SQLITE_OK ){
84
sqlite3_close(db);
85
return 0;
86
}
87
}
88
return db;
89
}
90
91
/*
92
** Attempt to construct a prepared statement for the cache database.
93
*/
94
static sqlite3_stmt *cacheStmt(sqlite3 *db, const char *zSql){
95
sqlite3_stmt *pStmt = 0;
96
int rc;
97
98
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
99
if( rc ){
100
sqlite3_finalize(pStmt);
101
pStmt = 0;
102
}
103
return pStmt;
104
}
105
106
/*
107
** This routine implements an SQL function that renders a large integer
108
** compactly: ex: 12.3MB
109
*/
110
static void cache_sizename(
111
sqlite3_context *context,
112
int argc,
113
sqlite3_value **argv
114
){
115
char zBuf[30];
116
double v, x;
117
assert( argc==1 );
118
v = sqlite3_value_double(argv[0]);
119
x = v<0.0 ? -v : v;
120
if( x>=1e9 ){
121
sqlite3_snprintf(sizeof(zBuf), zBuf, "%.1fGB", v/1e9);
122
}else if( x>=1e6 ){
123
sqlite3_snprintf(sizeof(zBuf), zBuf, "%.1fMB", v/1e6);
124
}else if( x>=1e3 ){
125
sqlite3_snprintf(sizeof(zBuf), zBuf, "%.1fKB", v/1e3);
126
}else{
127
sqlite3_snprintf(sizeof(zBuf), zBuf, "%gB", v);
128
}
129
sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT);
130
}
131
132
/*
133
** Register the sizename() SQL function with the SQLite database
134
** connection.
135
*/
136
static void cache_register_sizename(sqlite3 *db){
137
sqlite3_create_function(db, "sizename", 1, SQLITE_UTF8, 0,
138
cache_sizename, 0, 0);
139
}
140
141
/*
142
** Attempt to write pContent into the cache. If the cache file does
143
** not exist, then this routine is a no-op. Older cache entries might
144
** be deleted.
145
*/
146
void cache_write(Blob *pContent, const char *zKey){
147
sqlite3 *db;
148
sqlite3_stmt *pStmt;
149
int rc = 0;
150
int nKeep;
151
152
db = cacheOpen(0);
153
if( db==0 ) return;
154
sqlite3_busy_timeout(db, 10000);
155
sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, 0);
156
pStmt = cacheStmt(db, "INSERT INTO blob(data) VALUES(?1)");
157
if( pStmt==0 ) goto cache_write_end;
158
sqlite3_bind_blob(pStmt, 1, blob_buffer(pContent), blob_size(pContent),
159
SQLITE_STATIC);
160
if( sqlite3_step(pStmt)!=SQLITE_DONE ) goto cache_write_end;
161
sqlite3_finalize(pStmt);
162
pStmt = cacheStmt(db,
163
"INSERT OR IGNORE INTO cache(key,sz,tm,nref,id)"
164
"VALUES(?1,?2,strftime('%s','now'),1,?3)"
165
);
166
if( pStmt==0 ) goto cache_write_end;
167
sqlite3_bind_text(pStmt, 1, zKey, -1, SQLITE_STATIC);
168
sqlite3_bind_int(pStmt, 2, blob_size(pContent));
169
sqlite3_bind_int(pStmt, 3, sqlite3_last_insert_rowid(db));
170
if( sqlite3_step(pStmt)!=SQLITE_DONE) goto cache_write_end;
171
rc = sqlite3_changes(db);
172
173
/* If the write was successful, truncate the cache to keep at most
174
** max-cache-entry entries in the cache.
175
**
176
** The cache entry replacement algorithm is approximately LRU
177
** (least recently used). However, each access of an entry buys
178
** that entry an extra hour of grace, so that more commonly accessed
179
** entries are held in cache longer. The extra "grace" allotted to
180
** an entry is limited to 2 days worth.
181
*/
182
if( rc ){
183
nKeep = db_get_int("max-cache-entry",10);
184
sqlite3_finalize(pStmt);
185
pStmt = cacheStmt(db,
186
"DELETE FROM cache WHERE rowid IN ("
187
"SELECT rowid FROM cache"
188
" ORDER BY (tm + 3600*min(nRef,48)) DESC"
189
" LIMIT -1 OFFSET ?1)");
190
if( pStmt ){
191
sqlite3_bind_int(pStmt, 1, nKeep);
192
sqlite3_step(pStmt);
193
}
194
}
195
196
cache_write_end:
197
sqlite3_finalize(pStmt);
198
sqlite3_exec(db, rc ? "COMMIT" : "ROLLBACK", 0, 0, 0);
199
sqlite3_close(db);
200
}
201
202
/*
203
** SETTING: max-cache-entry width=10 default=10
204
**
205
** This is the maximum number of entries to allow in the web-cache
206
** for tarballs, ZIP-archives, and SQL-archives.
207
*/
208
209
/*
210
** Attempt to read content out of the cache with the given zKey. Return
211
** non-zero on success and zero if unable to locate the content.
212
**
213
** Possible reasons for returning zero:
214
** (1) This server does not implement a cache
215
** (2) The requested element is not in the cache
216
*/
217
int cache_read(Blob *pContent, const char *zKey){
218
sqlite3 *db;
219
sqlite3_stmt *pStmt;
220
int rc = 0;
221
222
db = cacheOpen(0);
223
if( db==0 ) return 0;
224
sqlite3_busy_timeout(db, 10000);
225
sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, 0);
226
pStmt = cacheStmt(db,
227
"SELECT blob.data FROM cache, blob"
228
" WHERE cache.key=?1 AND cache.id=blob.id");
229
if( pStmt==0 ) goto cache_read_done;
230
sqlite3_bind_text(pStmt, 1, zKey, -1, SQLITE_STATIC);
231
if( sqlite3_step(pStmt)==SQLITE_ROW ){
232
blob_append(pContent, sqlite3_column_blob(pStmt, 0),
233
sqlite3_column_bytes(pStmt, 0));
234
rc = 1;
235
sqlite3_reset(pStmt);
236
pStmt = cacheStmt(db,
237
"UPDATE cache SET nref=nref+1, tm=strftime('%s','now')"
238
" WHERE key=?1");
239
if( pStmt ){
240
sqlite3_bind_text(pStmt, 1, zKey, -1, SQLITE_STATIC);
241
sqlite3_step(pStmt);
242
}
243
}
244
sqlite3_finalize(pStmt);
245
cache_read_done:
246
sqlite3_exec(db, "COMMIT", 0, 0, 0);
247
sqlite3_close(db);
248
return rc;
249
}
250
251
/*
252
** Create a cache database for the current repository if no such
253
** database already exists.
254
*/
255
void cache_initialize(void){
256
sqlite3_close(cacheOpen(1));
257
}
258
259
/*
260
** COMMAND: cache* abbrv-subcom
261
**
262
** Usage: %fossil cache SUBCOMMAND
263
**
264
** Manage the cache used for potentially expensive web pages such as
265
** /zip and /tarball. SUBCOMMAND can be:
266
**
267
** clear Remove all entries from the cache.
268
**
269
** init Create the cache file if it does not already exist.
270
**
271
** list|ls List the keys and content sizes and other stats for
272
** all entries currently in the cache.
273
**
274
** size ?N? Query or set the maximum number of entries in the cache.
275
**
276
** status Show a summary of the cache status.
277
**
278
** The cache is stored in a file that is distinct from the repository
279
** but that is held in the same directory as the repository. The cache
280
** file can be deleted in order to completely disable the cache.
281
*/
282
void cache_cmd(void){
283
const char *zCmd;
284
int nCmd;
285
sqlite3 *db;
286
sqlite3_stmt *pStmt;
287
288
db_find_and_open_repository(0,0);
289
zCmd = g.argc>=3 ? g.argv[2] : "";
290
nCmd = (int)strlen(zCmd);
291
if( nCmd<=1 ){
292
fossil_fatal("Usage: %s cache SUBCOMMAND", g.argv[0]);
293
}
294
if( strncmp(zCmd, "init", nCmd)==0 ){
295
db = cacheOpen(0);
296
sqlite3_close(db);
297
if( db ){
298
fossil_print("cache already exists in file %z\n", cacheName());
299
}else{
300
db = cacheOpen(1);
301
sqlite3_close(db);
302
if( db ){
303
fossil_print("cache created in file %z\n", cacheName());
304
}else{
305
fossil_fatal("unable to create cache file %z", cacheName());
306
}
307
}
308
}else if( strncmp(zCmd, "clear", nCmd)==0 ){
309
db = cacheOpen(0);
310
if( db ){
311
sqlite3_exec(db, "DELETE FROM cache; DELETE FROM blob; VACUUM;",0,0,0);
312
sqlite3_close(db);
313
fossil_print("cache cleared\n");
314
}else{
315
fossil_print("nothing to clear; cache does not exist\n");
316
}
317
}else if( strncmp(zCmd, "list", nCmd)==0
318
|| strncmp(zCmd, "ls", nCmd)==0
319
|| strncmp(zCmd, "status", nCmd)==0
320
){
321
db = cacheOpen(0);
322
if( db==0 ){
323
fossil_print("cache does not exist\n");
324
}else{
325
int nEntry = 0;
326
char *zDbName = cacheName();
327
cache_register_sizename(db);
328
pStmt = cacheStmt(db,
329
"SELECT key, sizename(sz), nRef, datetime(tm,'unixepoch')"
330
" FROM cache"
331
" ORDER BY tm DESC"
332
);
333
if( pStmt ){
334
while( sqlite3_step(pStmt)==SQLITE_ROW ){
335
if( zCmd[0]=='l' ){
336
fossil_print("%s %4d %8s %s\n",
337
sqlite3_column_text(pStmt, 3),
338
sqlite3_column_int(pStmt, 2),
339
sqlite3_column_text(pStmt, 1),
340
sqlite3_column_text(pStmt, 0));
341
}
342
nEntry++;
343
}
344
sqlite3_finalize(pStmt);
345
}
346
sqlite3_close(db);
347
fossil_print(
348
"Filename: %s\n"
349
"Entries: %d\n"
350
"max-cache-entry: %d\n"
351
"Cache-file Size: %,lld\n",
352
zDbName,
353
nEntry,
354
db_get_int("max-cache-entry",10),
355
file_size(zDbName, ExtFILE)
356
);
357
fossil_free(zDbName);
358
}
359
}else if( strncmp(zCmd, "size", nCmd)==0 ){
360
if( g.argc>=4 ){
361
int n = atoi(g.argv[3]);
362
if( n>=5 ) db_set_int("max-cache-entry",n,0);
363
}
364
fossil_print("max-cache-entry: %d\n", db_get_int("max-cache-entry",10));
365
}else{
366
fossil_fatal("Unknown subcommand \"%s\"."
367
" Should be one of: clear init list size status", zCmd);
368
}
369
}
370
371
/*
372
** Given a cache key, find the check-in hash and return it as a separate
373
** string. The returned string is obtained from fossil_malloc() and must
374
** be freed by the caller.
375
**
376
** Return NULL if not found.
377
**
378
** The key is usually in a format like these:
379
**
380
** /tarball/HASH/NAME
381
** /zip/HASH/NAME
382
** /sqlar/HASH/NAME
383
*/
384
static char *cache_hash_of_key(const char *zKey){
385
int i;
386
if( zKey==0 ) return 0;
387
if( zKey[0]!='/' ) return 0;
388
zKey++;
389
while( zKey[0] && zKey[0]!='/' ) zKey++;
390
if( zKey[0]==0 ) return 0;
391
zKey++;
392
for(i=0; zKey[i] && zKey[i]!='/'; i++){}
393
if( !validate16(zKey, i) ) return 0;
394
return fossil_strndup(zKey, i);
395
}
396
397
398
/*
399
** WEBPAGE: cachestat
400
**
401
** Show information about the webpage cache. Requires Setup privilege.
402
*/
403
void cache_page(void){
404
sqlite3 *db = 0;
405
sqlite3_stmt *pStmt;
406
int doInit;
407
char *zDbName = cacheName();
408
int nEntry = 0;
409
int mxEntry = 0;
410
char zBuf[100];
411
412
login_check_credentials();
413
if( !g.perm.Setup ){ login_needed(0); return; }
414
style_set_current_feature("cache");
415
style_header("Web Cache Status");
416
style_submenu_element("Refresh","%R/cachestat");
417
doInit = P("init")!=0 && cgi_csrf_safe(2);
418
db = cacheOpen(doInit);
419
if( db!=0 ){
420
if( P("clear")!=0 && cgi_csrf_safe(2) ){
421
sqlite3_exec(db, "DELETE FROM cache; DELETE FROM blob; VACUUM;",0,0,0);
422
}
423
cache_register_sizename(db);
424
pStmt = cacheStmt(db,
425
"SELECT key, sz, nRef, datetime(tm,'unixepoch')"
426
" FROM cache"
427
" ORDER BY (tm + 3600*min(nRef,48)) DESC"
428
);
429
if( pStmt ){
430
while( sqlite3_step(pStmt)==SQLITE_ROW ){
431
const unsigned char *zName = sqlite3_column_text(pStmt,0);
432
char *zHash = cache_hash_of_key((const char*)zName);
433
if( nEntry==0 ){
434
@ <h2>Current Cache Entries:</h2>
435
@ <ol>
436
}
437
@ <li><p>%z(href("%R/cacheget?key=%T",zName))%h(zName)</a><br>
438
@ size: %,lld(sqlite3_column_int64(pStmt,1)),
439
@ hit-count: %d(sqlite3_column_int(pStmt,2)),
440
@ last-access: %s(sqlite3_column_text(pStmt,3))Z \
441
if( zHash ){
442
@ &rarr; %z(href("%R/timeline?c=%S",zHash))check-in info</a>\
443
fossil_free(zHash);
444
}
445
@ </p></li>
446
nEntry++;
447
}
448
sqlite3_finalize(pStmt);
449
if( nEntry ){
450
@ </ol>
451
}
452
}
453
}
454
@ <h2>About The Web-Cache</h2>
455
@ <p>
456
@ The web-cache is a separate database file that holds cached copies
457
@ tarballs, ZIP archives, and other pages that are expensive to compute
458
@ and are likely to be reused.
459
@ <form method="post">
460
login_insert_csrf_secret();
461
@ <ul>
462
if( db==0 ){
463
@ <li> Web-cache is currently disabled.
464
@ <input type="submit" name="init" value="Enable">
465
}else{
466
bigSizeName(sizeof(zBuf), zBuf, file_size(zDbName, ExtFILE));
467
mxEntry = db_get_int("max-cache-entry",10);
468
@ <li> Filename of the cache database: <b>%h(zDbName)</b>
469
@ <li> Size of the cache database: %s(zBuf)
470
@ <li> Maximum number of entries: %d(mxEntry)
471
@ <li> Number of cache entries used: %d(nEntry)
472
@ <li> Change the max-cache-entry setting on the
473
@ <a href="%R/setup_settings">Settings</a> page to adjust the
474
@ maximum number of entries in the cache.
475
@ <li><input type="submit" name="clear" value="Clear the cache">
476
@ <li> Disable the cache by manually deleting the cache database file.
477
}
478
@ </ul>
479
@ </form>
480
fossil_free(zDbName);
481
if( db ) sqlite3_close(db);
482
style_finish_page();
483
}
484
485
/*
486
** WEBPAGE: cacheget
487
**
488
** Usage: /cacheget?key=KEY
489
**
490
** Download a single entry for the cache, identified by KEY.
491
** This page is normally a hyperlink from the /cachestat page.
492
** Requires Admin privilege.
493
*/
494
void cache_getpage(void){
495
const char *zKey;
496
Blob content;
497
498
login_check_credentials();
499
if( !g.perm.Setup ){ login_needed(0); return; }
500
zKey = PD("key","");
501
blob_zero(&content);
502
if( cache_read(&content, zKey)==0 ){
503
style_set_current_feature("cache");
504
style_header("Cache Download Error");
505
@ The cache does not contain any entry with this key: "%h(zKey)"
506
style_finish_page();
507
return;
508
}
509
cgi_set_content(&content);
510
cgi_set_content_type("application/x-compressed");
511
}
512

Keyboard Shortcuts

Open search /
Next entry (timeline) j
Previous entry (timeline) k
Open focused entry Enter
Show this help ?
Toggle theme Top nav button