14 #include "XrdVersion.hh"
24 m_allow_xrdpfc_command(false),
25 m_data_space(
"public"),
26 m_meta_space(
"public"),
30 m_fileUsageBaseline(-1),
31 m_fileUsageNominal(-1),
34 m_purgeColdFilesAge(-1),
35 m_purgeAgeBasedPeriod(10),
37 m_dirStatsMaxDepth(-1),
38 m_dirStatsStoreDepth(0),
39 m_bufferSize(128*1024),
41 m_RamKeepStdBlocks(0),
44 m_prefetch_max_blocks(10),
45 m_hdfsbsize(128*1024*1024),
50 m_onlyIfCachedMinSize(1024*1024),
51 m_onlyIfCachedMinFrac(1.0)
55 bool Cache::cfg2bytes(
const std::string &str,
long long &store,
long long totalSpace,
const char *name)
58 snprintf(errStr, 1024,
"ConfigParameters() Error parsing parameter %s", name);
60 if (::isalpha(*(str.rbegin())))
71 double frac = strtod(str.c_str(), &eP);
72 if (errno || eP == str.c_str())
74 m_log.
Emsg(errStr, str.c_str());
78 store =
static_cast<long long>(totalSpace * frac + 0.5);
81 if (store < 0 || store > totalSpace)
83 snprintf(errStr, 1024,
"ConfigParameters() Error: parameter %s should be between 0 and total available disk space (%lld) - it is %lld (given as %s)",
84 name, totalSpace, store, str.c_str());
85 m_log.
Emsg(errStr,
"");
113 const char *val, *val2;
114 struct cschkopts {
const char *opname;
int opval;} csopts[] =
121 int i, numopts =
sizeof(csopts)/
sizeof(
struct cschkopts);
124 if (! (val =
Config.GetWord()))
125 {m_log.
Emsg(
"Config",
"cschk parameter not specified");
return false; }
129 if ((
isNo = strncmp(val,
"no", 2) == 0))
133 for (i = 0; i < numopts; i++)
135 if (!strcmp(val2, csopts[i].opname))
138 m_configuration.
m_cs_Chk &= ~csopts[i].opval;
139 else if (csopts[i].opval)
140 m_configuration.
m_cs_Chk |= csopts[i].opval;
142 m_configuration.
m_cs_Chk = csopts[i].opval;
148 if (strcmp(val,
"uvkeep"))
150 m_log.
Emsg(
"Config",
"invalid cschk option -", val);
153 if (!(val =
Config.GetWord()))
155 m_log.
Emsg(
"Config",
"cschk uvkeep value not specified");
158 if (!strcmp(val,
"lru"))
171 m_configuration.m_cs_ChkTLS = m_configuration.m_cs_Chk &
CSChk_TLS;
174 m_env->Put(
"psx.CSNet", m_configuration.is_cschk_net() ? (m_configuration.m_cs_ChkTLS ?
"2" :
"1") :
"0");
195 if (! (val =
Config.GetWord()) || ! val[0])
197 TRACE(
Info,
" Cache::Config() decisionlib not specified; always caching files");
207 Config.GetRest(params, 4096);
216 if (! ep) {myLib->
Unload(
true);
return false; }
221 TRACE(
Error,
"Config() decisionlib was not able to create a decision object");
227 m_decisionpoints.push_back(d);
239 static struct traceopts {
const char *opname;
int opval; } tropts[] =
249 int numopts =
sizeof(tropts)/
sizeof(
struct traceopts);
251 if (! (val =
Config.GetWord()))
252 {m_log.
Emsg(
"Config",
"trace option not specified");
return 1; }
254 for (
int i = 0; i < numopts; i++)
256 if (! strcmp(val, tropts[i].opname))
258 m_trace->
What = tropts[i].opval;
262 m_log.
Emsg(
"Config",
"invalid trace option -", val);
276 const char *theINS = getenv(
"XRDINSTANCE");
277 m_isClient = (theINS != 0 && strncmp(
"*client ", theINS, 8) == 0);
285 if (! config_filename || ! *config_filename)
287 TRACE(
Error,
"Config() configuration file not specified.");
292 if ( (fd =
open(config_filename, O_RDONLY, 0)) < 0)
294 TRACE(
Error,
"Config() can't open configuration file " << config_filename);
299 static const char *cvec[] = {
"*** pfc plugin config:", 0 };
305 if (! ofsCfg)
return false;
321 bool retval =
true, aOK =
true;
323 while ((var =
Config.GetMyFirstWord()))
325 if (! strcmp(var,
"pfc.osslib"))
329 else if (! strcmp(var,
"pfc.cschk"))
333 else if (! strcmp(var,
"pfc.decisionlib"))
337 else if (! strcmp(var,
"pfc.trace"))
341 else if (! strcmp(var,
"pfc.allow_xrdpfc_command"))
345 else if (! strncmp(var,
"pfc.", 4))
347 retval = ConfigParameters(std::string(var+4),
Config, tmpc);
360 myEnv.
Put(
"oss.runmode",
"pfc");
364 if (snprintf(csi_conf, 128,
"space=%s nofill", m_configuration.
m_meta_space.c_str()) < 128)
368 TRACE(
Error,
"Config() buffer too small for libXrdOssCsi params.");
378 TRACE(
Error,
"Config() Unable to create an OSS object");
387 m_log.
Emsg(
"ConfigParameters()",
"error obtaining stat info for data space ", m_configuration.
m_data_space.c_str());
390 if (sP.
Total < 10ll << 20)
392 m_log.
Emsg(
"ConfigParameters()",
"available data space is less than 10 MB (can be due to a mistake in oss.localroot directive) for space ",
403 m_log.
Emsg(
"ConfigParameters()",
"pfc.diskusage should have lowWatermark < highWatermark.");
419 m_log.
Emsg(
"ConfigParameters()",
"pfc.diskusage files should have baseline < nominal < max.");
452 m_configuration.
m_RamAbsAvailable = m_isClient ? 256ll * 1024 * 1024 : 1024ll * 1024 * 1024;
454 snprintf(buff,
sizeof(buff),
"RAM usage pfc.ram is not specified. Default value %s is used.", m_isClient ?
"256m" :
"1g");
455 m_log.
Say(
"Config info: ", buff);
462 char* cenv = getenv(
"XRDDEBUG");
463 if (cenv && ! strcmp(cenv,
"1") && m_trace->
What < 4) m_trace->
What = 4;
469 const char *csc[] = {
"off",
"cache nonet",
"nocache net notls",
473 "off",
"cache nonet",
"nocache net tls",
476 char buff[8192], uvk[32];
480 sprintf(uvk,
"%lld", (
long long) m_configuration.
m_cs_UVKeep);
482 loff = snprintf(buff,
sizeof(buff),
"Config effective %s pfc configuration:\n"
483 " pfc.cschk %s uvkeep %s\n"
484 " pfc.blocksize %lld\n"
487 " pfc.writequeue %d %d\n"
488 " # Total available disk: %lld\n"
489 " pfc.diskusage %lld %lld files %lld %lld %lld purgeinterval %d purgecoldfiles %d\n"
490 " pfc.spaces %s %s\n"
493 " pfc.acchistorysize %d\n"
494 " pfc.onlyIfCachedMinBytes %lld\n"
495 " pfc.onlyIfCachedMinFrac %.2f\n",
497 csc[
int(m_configuration.
m_cs_Chk)], uvk,
516 loff += snprintf(buff + loff,
sizeof(buff) - loff,
517 " pfc.dirstats maxdepth %d ((internal: store_depth %d, size_of_dirlist %d, size_of_globlist %d))\n",
520 loff += snprintf(buff + loff,
sizeof(buff) - loff,
" dirlist:\n");
522 loff += snprintf(buff + loff,
sizeof(buff) - loff,
" %s\n", i->c_str());
523 loff += snprintf(buff + loff,
sizeof(buff) - loff,
" globlist:\n");
525 loff += snprintf(buff + loff,
sizeof(buff) - loff,
" %s/*\n", i->c_str());
530 loff += snprintf(buff + loff,
sizeof(buff) - loff,
" pfc.hdfsmode hdfsbsize %lld\n", m_configuration.
m_hdfsbsize);
541 loff += snprintf(buff + loff,
sizeof(buff) - loff,
" pfc.user %s\n", m_configuration.
m_username.c_str());
546 m_env->
Put(
"XRDPFC.SEGSIZE", std::to_string(m_configuration.
m_bufferSize).c_str());
555 m_log.
Say(
"Config Proxy File Cache g-stream has", m_gstream ?
"" :
" NOT",
" been configured via xrootd.monitor directive");
557 m_log.
Say(
"------ Proxy File Cache configuration parsing ", aOK ?
"completed" :
"failed");
559 if (ofsCfg)
delete ofsCfg;
563 #ifdef XRDPFC_CKSUM_TEST
569 Info::TestCksumStuff();
583 struct ConfWordGetter
588 ConfWordGetter(
XrdOucStream& c) : m_config(c), m_last_word((char*)1) {}
590 const char* GetWord() {
if (HasLast()) m_last_word = m_config.
GetWord();
return HasLast() ? m_last_word :
""; }
591 bool HasLast() {
return (m_last_word != 0); }
594 ConfWordGetter cwg(config);
597 if ( part ==
"user" )
600 if ( ! cwg.HasLast())
602 m_log.
Emsg(
"Config",
"Error: pfc.user requires a parameter.");
606 else if ( part ==
"diskusage" )
613 m_log.
Emsg(
"Config",
"Error: pfc.diskusage parameter requires at least two arguments.");
618 while ((p = cwg.GetWord()) && cwg.HasLast())
620 if (strcmp(p,
"files") == 0)
626 if ( ! cwg.HasLast())
628 m_log.
Emsg(
"Config",
"Error: pfc.diskusage files directive requires three arguments.");
632 else if (strcmp(p,
"sleep") == 0 || strcmp(p,
"purgeinterval") == 0)
634 if (strcmp(p,
"sleep") == 0) m_log.
Emsg(
"Config",
"warning sleep directive is deprecated in pfc.diskusage. Please use purgeinterval instead.");
641 else if (strcmp(p,
"purgecoldfiles") == 0)
654 m_log.
Emsg(
"Config",
"Error: diskusage stanza contains unknown directive", p);
658 else if ( part ==
"acchistorysize" )
665 else if ( part ==
"dirstats" )
668 while ((p = cwg.GetWord()) && cwg.HasLast())
670 if (strcmp(p,
"maxdepth") == 0)
678 else if (strcmp(p,
"dir") == 0)
681 if (p && p[0] ==
'/')
685 char d[1024]; d[0] = 0;
693 if (*(pd - 1) ==
'/')
705 if (*pd ==
'/' && pd != d) *pd = 0;
708 if (ld >= 2 && d[ld-1] ==
'*' && d[ld-2] ==
'/')
713 printf(
"Glob %s -> %s -- depth = %d\n", p, d, depth);
718 printf(
"Dir %s -> %s -- depth = %d\n", p, d, depth);
725 m_log.
Emsg(
"Config",
"Error: dirstats dir parameter requires a directory argument starting with a '/'.");
731 m_log.
Emsg(
"Config",
"Error: dirstats stanza contains unknown directive '", p,
"'");
736 else if ( part ==
"blocksize" )
738 long long minBSize = 4 * 1024;
739 long long maxBSize = 512 * 1024 * 1024;
748 m_log.
Emsg(
"Config",
"pfc.blocksize must be a multiple of 4 kB. Rounded up.");
751 else if ( part ==
"prefetch" || part ==
"nramprefetch" )
753 if (part ==
"nramprefetch")
755 m_log.
Emsg(
"Config",
"pfc.nramprefetch is deprecated, please use pfc.prefetch instead. Replacing the directive internally.");
764 else if ( part ==
"nramread" )
766 m_log.
Emsg(
"Config",
"pfc.nramread is deprecated, please use pfc.ram instead. Ignoring this directive.");
769 else if ( part ==
"ram" )
771 long long minRAM = m_isClient ? 256 * 1024 * 1024 : 1024 * 1024 * 1024;
772 long long maxRAM = 256 * minRAM;
778 else if ( part ==
"writequeue")
789 else if ( part ==
"spaces" )
793 if ( ! cwg.HasLast())
795 m_log.
Emsg(
"Config",
"spacenames requires two parameters: <data-space> <metadata-space>.");
799 else if ( part ==
"hdfsmode" )
801 m_log.
Emsg(
"Config",
"pfc.hdfsmode is currently unsupported.");
806 const char* params = cwg.GetWord();
809 if (! strncmp(
"hdfsbsize", params, 9))
811 long long minBlSize = 32 * 1024;
812 long long maxBlSize = 128 * 1024 * 1024;
813 if (
XrdOuca2x::a2sz(m_log,
"Error getting file fragment size", cwg.GetWord(), &m_configuration.
m_hdfsbsize, minBlSize, maxBlSize))
820 m_log.
Emsg(
"Config",
"Error setting the fragment size parameter name");
825 else if ( part ==
"flush" )
828 if ( ! cwg.HasLast())
830 m_log.
Emsg(
"Config",
"Error: pfc.flush requires a parameter.");
834 else if ( part ==
"onlyifcached" )
837 while ((p = cwg.GetWord()) && cwg.HasLast())
839 if (strcmp(p,
"minsize") == 0)
841 std::string minBytes = cwg.GetWord();
842 long long minBytesTop = 1024 * 1024 * 1024;
843 if (::isalpha(*(minBytes.rbegin())))
858 if (strcmp(p,
"minfrac") == 0)
860 std::string minFrac = cwg.GetWord();
863 double frac = strtod(minFrac.c_str(), &eP);
864 if (errno || eP == minFrac.c_str())
866 m_log.
Emsg(
"Config",
"Error setting fraction for only-if-cached directive");
873 m_log.
Emsg(
"Config",
"Error: onlyifcached stanza contains unknown directive", p);
879 m_log.
Emsg(
"ConfigParameters() unmatched pfc parameter", part.c_str());
XrdVERSIONINFO(XrdOucGetCache, XrdPfc)
XrdOucCache * XrdOucGetCache(XrdSysLogger *logger, const char *config_filename, const char *parameters, XrdOucEnv *env)
int open(const char *path, int oflag,...)
int isNo(int dflt, const char *Msg1, const char *Msg2, const char *Msg3)
bool Plugin(XrdAccAuthorize *&piP)
Get Authorization plugin.
static XrdOfsConfigPI * New(const char *cfn, XrdOucStream *cfgP, XrdSysError *errP, XrdVersionInfo *verP=0, XrdSfsFileSystem *sfsP=0)
bool Load(int what, XrdOucEnv *envP=0)
bool Push(TheLib what, const char *plugP, const char *parmP=0)
virtual int StatVS(XrdOssVSInfo *vsP, const char *sname=0, int updt=0)
static int Export(const char *Var, const char *Val)
void * GetPtr(const char *varname)
void Put(const char *varname, const char *value)
void * Resolve(const char *symbl, int mcnt=1)
void Unload(bool dodel=false)
char * GetWord(int lowcase=0)
static int UserName(uid_t uID, char *uName, int uNsz)
static int a2i(XrdSysError &, const char *emsg, const char *item, int *val, int minv=-1, int maxv=-1)
static int a2sz(XrdSysError &, const char *emsg, const char *item, long long *val, long long minv=-1, long long maxv=-1)
static int a2ll(XrdSysError &, const char *emsg, const char *item, long long *val, long long minv=-1, long long maxv=-1)
static int a2tm(XrdSysError &, const char *emsg, const char *item, int *val, int minv=-1, int maxv=-1)
bool Config(const char *config_filename, const char *parameters)
Parse configuration file.
Base class for selecting which files should be cached.
virtual bool ConfigDecision(const char *params)
Status of cached file. Can be read from and written into a binary file.
static size_t s_maxNumAccess
int Emsg(const char *esfx, int ecode, const char *text1, const char *text2=0)
void Say(const char *text1, const char *text2=0, const char *txt3=0, const char *text4=0, const char *text5=0, const char *txt6=0)
long long m_hdfsbsize
used with m_hdfsmode, default 128MB
long long m_RamAbsAvailable
available from configuration
long long m_flushCnt
nuber of unsynced blcoks on disk before flush is called
int m_accHistorySize
max number of entries in access history part of cinfo file
int m_wqueue_threads
number of threads writing blocks to disk
long long m_diskTotalSpace
total disk space on configured partition or oss space
long long m_fileUsageMax
cache purge - files usage maximum
long long m_fileUsageBaseline
cache purge - files usage baseline
int m_dirStatsStoreDepth
depth to which statistics should be collected
bool m_allow_xrdpfc_command
flag for enabling access to /xrdpfc-command/ functionality.
long long m_diskUsageHWM
cache purge - disk usage high water mark
bool is_cschk_cache() const
std::set< std::string > m_dirStatsDirGlobs
directory globs for which stat reporting was requested
int m_prefetch_max_blocks
maximum number of blocks to prefetch per file
bool m_cs_ChkTLS
Allow TLS.
long long m_fileUsageNominal
cache purge - files usage nominal
int m_cs_Chk
Checksum check.
int m_purgeAgeBasedPeriod
peform cold file / uvkeep purge every this many purge cycles
bool m_hdfsmode
flag for enabling block-level operation
int m_purgeColdFilesAge
purge files older than this age
std::string m_data_space
oss space for data files
std::set< std::string > m_dirStatsDirs
directories for which stat reporting was requested
long long m_diskUsageLWM
cache purge - disk usage low water mark
int m_RamKeepStdBlocks
number of standard-sized blocks kept after release
long long m_bufferSize
prefetch buffer size, default 1MB
std::string m_meta_space
oss space for metadata files (cinfo)
int m_wqueue_blocks
maximum number of blocks written per write-queue loop
std::string m_username
username passed to oss plugin
bool is_cschk_net() const
double m_onlyIfCachedMinFrac
minimum fraction of downloaded file, used by only-if-cached CGI option
time_t m_cs_UVKeep
unverified checksum cache keep
int m_dirStatsMaxDepth
maximum depth for statistics write out
int m_purgeInterval
sleep interval between cache purges
long long m_onlyIfCachedMinSize
minumum size of downloaded file, used by only-if-cached CGI option
bool is_dir_stat_reporting_on() const
std::string m_diskUsageLWM
std::string m_diskUsageHWM
std::string m_fileUsageBaseline
std::string m_fileUsageNominal
std::string m_fileUsageMax