35 #include <sys/types.h>
58 #define NO_LM_EXPIRATION 24*3600 // 24 hours
59 #define MAX_LM_EXPIRATION 48*3600 // Max expiration from LM
64 #define LM_EXPIRATION(t) (min((MAX_LM_EXPIRATION), static_cast<int>((t) / 10)))
81 for (
const char *ptr = url.c_str(); *ptr; ptr++)
87 HTTPCacheTable::HTTPCacheTable(
const string &cache_root,
int block_size) :
88 d_cache_root(cache_root), d_block_size(block_size), d_current_size(0), d_new_entries(0)
107 DBG2(cerr <<
"Deleting CacheEntry: " << e << endl);
117 for_each(cp->begin(), cp->end(), delete_cache_entry);
120 delete get_cache_table()[i];
121 get_cache_table()[i] = 0;
125 delete[] d_cache_table;
135 class DeleteExpired :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
141 d_time(t), d_table(table) {
146 void operator()(HTTPCacheTable::CacheEntry *&e) {
147 if (e && !e->readers && (e->freshness_lifetime
148 < (e->corrected_initial_age + (d_time - e->response_time)))) {
149 DBG(cerr <<
"Deleting expired cache entry: " << e->url << endl);
150 d_table.remove_cache_entry(e);
162 for_each(slot->begin(), slot->end(), DeleteExpired(*
this, time));
163 slot->erase(
remove(slot->begin(), slot->end(),
175 class DeleteByHits :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
181 d_table(table), d_hits(hits) {
184 void operator()(HTTPCacheTable::CacheEntry *&e) {
185 if (e && !e->readers && e->hits <= d_hits) {
186 DBG(cerr <<
"Deleting cache entry: " << e->url << endl);
187 d_table.remove_cache_entry(e);
196 if (get_cache_table()[cnt]) {
198 for_each(slot->begin(), slot->end(), DeleteByHits(*
this, hits));
199 slot->erase(
remove(slot->begin(), slot->end(),
211 class DeleteBySize :
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
217 d_table(table), d_size(size) {
220 void operator()(HTTPCacheTable::CacheEntry *&e) {
221 if (e && !e->readers && e->size > d_size) {
222 DBG(cerr <<
"Deleting cache entry: " << e->url << endl);
223 d_table.remove_cache_entry(e);
231 if (get_cache_table()[cnt]) {
233 for_each(slot->begin(), slot->end(), DeleteBySize(*
this, size));
234 slot->erase(
remove(slot->begin(), slot->end(),
274 FILE *fp = fopen(d_cache_index.c_str(),
"r");
282 while (!feof(fp) && fgets(line, 1024, fp)) {
284 DBG2(cerr << line << endl);
287 int res = fclose(fp) ;
289 DBG(cerr <<
"HTTPCache::cache_index_read - Failed to close " << (
void *)fp << endl);
309 istringstream iss(line);
311 iss >> entry->cachename;
318 iss >> entry->expires;
324 iss >> entry->freshness_lifetime;
325 iss >> entry->response_time;
326 iss >> entry->corrected_initial_age;
328 iss >> entry->must_revalidate;
335 class WriteOneCacheEntry :
336 public unary_function<HTTPCacheTable::CacheEntry *, void>
342 WriteOneCacheEntry(FILE *fp) : d_fp(fp)
345 void operator()(HTTPCacheTable::CacheEntry *e)
347 if (e && fprintf(d_fp,
348 "%s %s %s %ld %ld %ld %c %d %d %ld %ld %ld %c\r\n",
350 e->cachename.c_str(),
355 e->range ?
'1' :
'0',
358 (long)(e->freshness_lifetime),
359 (long)(e->response_time),
360 (long)(e->corrected_initial_age),
361 e->must_revalidate ?
'1' :
'0') < 0)
362 throw Error(
"Cache Index. Error writing cache index\n");
378 DBG(cerr <<
"Cache Index. Writing index " << d_cache_index << endl);
382 if ((fp = fopen(d_cache_index.c_str(),
"wb")) == NULL) {
383 throw Error(
string(
"Cache Index. Can't open `") + d_cache_index
384 +
string(
"' for writing"));
393 for_each(cp->begin(), cp->end(), WriteOneCacheEntry(fp));
397 int res = fclose(fp);
399 DBG(cerr <<
"HTTPCache::cache_index_write - Failed to close "
400 << (
void *)fp << endl);
423 struct stat stat_info;
426 path << d_cache_root << hash;
427 string p = path.str();
429 if (stat(p.c_str(), &stat_info) == -1) {
430 DBG2(cerr <<
"Cache....... Create dir " << p << endl);
431 if (
MKDIR(p.c_str(), 0777) < 0) {
432 DBG2(cerr <<
"Cache....... Can't create..." << endl);
433 throw Error(
"Could not create cache slot to hold response! Check the write permissions on your disk cache directory. Cache root: " + d_cache_root +
".");
437 DBG2(cerr <<
"Cache....... Directory " << p <<
" already exists"
463 hash_dir +=
"\\dodsXXXXXX";
465 hash_dir +=
"/dodsXXXXXX";
470 vector<char> templat(hash_dir.size() + 1);
471 strncpy(&templat[0], hash_dir.c_str(), hash_dir.size() + 1);
485 throw Error(
"The HTTP Cache could not create a file to hold the response; it will not be cached.");
488 entry->cachename = &templat[0];
496 entry_disk_space(
int size,
unsigned int block_size)
498 unsigned int num_of_blocks = (size + block_size) / block_size;
500 DBG(cerr <<
"size: " << size <<
", block_size: " << block_size
501 <<
", num_of_blocks: " << num_of_blocks << endl);
503 return num_of_blocks * block_size;
518 int hash = entry->hash;
520 if (!d_cache_table[hash])
523 d_cache_table[hash]->push_back(entry);
525 DBG(cerr <<
"add_entry_to_cache_table, current_size: " << d_current_size
526 <<
", entry->size: " << entry->size <<
", block size: " << d_block_size
529 d_current_size += entry_disk_space(entry->size, d_block_size);
531 DBG(cerr <<
"add_entry_to_cache_table, current_size: " << d_current_size << endl);
540 HTTPCacheTable::get_locked_entry_from_cache_table(
const string &url)
542 return get_locked_entry_from_cache_table(
get_hash(url), url);
553 HTTPCacheTable::get_locked_entry_from_cache_table(
int hash,
const string &url)
555 DBG(cerr <<
"url: " << url <<
"; hash: " << hash << endl);
556 DBG(cerr <<
"d_cache_table: " << hex << d_cache_table << dec << endl);
557 if (d_cache_table[hash]) {
562 if ((*i) && (*i)->url == url) {
563 (*i)->lock_read_response();
578 HTTPCacheTable::CacheEntry *
582 if (d_cache_table[hash]) {
587 if ((*i) && (*i)->url == url) {
588 (*i)->lock_write_response();
610 throw InternalErr(__FILE__, __LINE__,
"Tried to delete a cache entry that is in use.");
612 REMOVE(entry->cachename.c_str());
617 unsigned int eds = entry_disk_space(entry->size,
get_block_size());
625 class DeleteCacheEntry:
public unary_function<HTTPCacheTable::CacheEntry *&, void>
632 : d_url(url), d_cache_table(c)
635 void operator()(HTTPCacheTable::CacheEntry *&e)
637 if (e && e->url == d_url) {
638 e->lock_write_response();
639 d_cache_table->remove_cache_entry(e);
640 e->unlock_write_response();
656 if (d_cache_table[hash]) {
658 for_each(cp->begin(), cp->end(), DeleteCacheEntry(
this, url));
666 class DeleteUnlockedCacheEntry:
public unary_function<HTTPCacheTable::CacheEntry *&, void> {
674 void operator()(HTTPCacheTable::CacheEntry *&e)
677 d_table.remove_cache_entry(e);
691 for_each(slot->begin(), slot->end(), DeleteUnlockedCacheEntry(*
this));
715 entry->response_time = time(NULL);
716 time_t apparent_age = max(0, static_cast<int>(entry->response_time - entry->date));
717 time_t corrected_received_age = max(apparent_age, entry->age);
718 time_t response_delay = entry->response_time - request_time;
719 entry->corrected_initial_age = corrected_received_age + response_delay;
724 time_t freshness_lifetime = entry->max_age;
725 if (freshness_lifetime < 0) {
726 if (entry->expires < 0) {
728 freshness_lifetime = default_expiration;
735 freshness_lifetime = entry->expires - entry->date;
738 entry->freshness_lifetime = max(0, static_cast<int>(freshness_lifetime));
740 DBG2(cerr <<
"Cache....... Received Age " << entry->age
741 <<
", corrected " << entry->corrected_initial_age
742 <<
", freshness lifetime " << entry->freshness_lifetime << endl);
757 const vector<string> &headers)
759 vector<string>::const_iterator i;
760 for (i = headers.begin(); i != headers.end(); ++i) {
765 string::size_type colon = (*i).find(
':');
768 if (colon == string::npos)
771 string header = (*i).substr(0, (*i).find(
':'));
772 string value = (*i).substr((*i).find(
": ") + 2);
773 DBG2(cerr <<
"Header: " << header << endl);
DBG2(cerr <<
"Value: " << value << endl);
775 if (header ==
"ETag") {
778 else if (header ==
"Last-Modified") {
781 else if (header ==
"Expires") {
784 else if (header ==
"Date") {
787 else if (header ==
"Age") {
790 else if (header ==
"Content-Length") {
791 unsigned long clength = strtoul(value.c_str(), 0, 0);
792 if (clength > max_entry_size)
795 else if (header ==
"Cache-Control") {
799 if (value ==
"no-cache" || value ==
"no-store")
804 else if (value ==
"must-revalidate")
805 entry->must_revalidate =
true;
806 else if (value.find(
"max-age") != string::npos) {
807 string max_age = value.substr(value.find(
"=" + 1));
819 d_locked_entries[body] = entry;
826 throw InternalErr(
"There is no cache entry for the response given.");
828 d_locked_entries.erase(body);
831 if (entry->readers < 0)
832 throw InternalErr(
"An unlocked entry was released");
836 return !d_locked_entries.empty();