restored/worklog.inc
← All source files · Download this file
18,039 bytes · SHA256 2bad1260cee52eb39eb34cdeccdde90c96e5cc0fa1ce5968ebaa646488343a77
// Bounded work-log facilities. Included after utilities.inc.
static std::size_t variableSize(const variable& v) {
std::size_t n=sizeof(variable)+v.value.capacity()+v.structuredBytes+v.items.capacity()*sizeof(std::string);
for(const auto& x:v.items) n+=x.capacity();
return n;
}
static variable typedValue(Frame& f,const Token& t,const Source& s) {
if(!t.literal && !t.text.empty() && t.text[0]=='$') return lookup(f,t,s);
if(!t.literal && !t.text.empty() && t.text[0]=='%') {
auto digits=t.text.substr(1);
if(digits.empty() || digits.size()>4 || digits.find_first_not_of("0123456789")!=std::string::npos) fail(s,"invalid argument index");
auto i=static_cast<std::size_t>(integer(digits,1,9999,s));
if(i>f.args.size()) fail(s,"argument index out of bounds");
return f.args[i-1];
}
return variable{value(f,t,s),{},false};
}
static void worklogValidate(const CodeLine& c) {
const auto& a=c.arg; const auto& s=c.source; count(c,2,6);
auto op=a[0].text, sub=a[1].text;
if(op=="csv") {
count(c,4,4); destination(a[2],s);
if(sub!="parse" && sub!="stringify") fail(s,"unknown csv operation");
if(sub=="stringify") destination(a[3],s);
} else if(op=="date") {
count(c,5,5); destination(a[2],s);
if(sub!="parse" && sub!="format" && sub!="diff") fail(s,"unknown date operation");
} else if(op=="db") {
if(sub=="open") { count(c,4,4); destination(a[2],s); }
else if(sub=="close" || sub=="begin" || sub=="commit" || sub=="rollback") count(c,3,3);
else if(sub=="exec" || sub=="query") { count(c,6,6); destination(a[2],s); destination(a[5],s); }
else fail(s,"unknown db operation");
}
}
static int dateOffset(const std::string& zone,const Source& s) {
if(zone=="UTC" || zone=="Z") return 0;
auto digit=[](char c) { return c>='0' && c<='9'; };
if(zone.size()!=6 || (zone[0]!='+' && zone[0]!='-') || zone[3]!=':' ||
!digit(zone[1]) || !digit(zone[2]) || !digit(zone[4]) || !digit(zone[5])) fail(s,"date zone must be UTC or fixed +/-HH:MM");
int h=(zone[1]-'0')*10+zone[2]-'0', m=(zone[4]-'0')*10+zone[5]-'0';
if(h>14 || m>59 || (h==14 && m!=0)) fail(s,"date offset out of range");
return (zone[0]=='-' ? -1 : 1)*(h*3600+m*60);
}
static long long dateParse(const std::string& text,int offset,const Source& s) {
if(text.size()!=10 && text.size()!=19) fail(s,"date requires YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS");
for(std::size_t i=0;i<text.size();++i) {
char expected=i==4 || i==7 ? '-' : i==10 ? 'T' : i==13 || i==16 ? ':' : 0;
if(expected ? text[i]!=expected : text[i]<'0' || text[i]>'9') fail(s,"malformed date");
}
auto n=[&](std::size_t i,std::size_t len) { return std::stoi(text.substr(i,len)); };
int y=n(0,4), m=n(5,2), d=n(8,2), h=0, min=0, sec=0;
if(text.size()==19) { h=n(11,2); min=n(14,2); sec=n(17,2); }
static const int days[]={31,28,31,30,31,30,31,31,30,31,30,31};
bool leap=y%4==0 && (y%100!=0 || y%400==0);
if(y<1 || m<1 || m>12 || d<1 || d>days[m-1]+(m==2 && leap ? 1 : 0) || h>23 || min>59 || sec>59) fail(s,"date component out of range");
std::tm tm{}; tm.tm_year=y-1900; tm.tm_mon=m-1; tm.tm_mday=d; tm.tm_hour=h; tm.tm_min=min; tm.tm_sec=sec;
return static_cast<long long>(timegm(&tm))-offset;
}
static std::string dateFormat(long long epoch,int offset,const Source& s) {
auto t=static_cast<std::time_t>(epoch+offset); std::tm tm{};
if(!gmtime_r(&t,&tm) || tm.tm_year < -1899 || tm.tm_year>8099) fail(s,"date result outside years 0001..9999");
char b[32]; if(!std::strftime(b,sizeof(b),"%Y-%m-%dT%H:%M:%S",&tm)) fail(s,"date format failed");
// strftime does not pad early years consistently across libc implementations.
char year[16]; std::snprintf(year,sizeof(year),"%04d",tm.tm_year+1900);
auto result=std::string(b); auto dash=result.find('-'); return std::string(year)+result.substr(dash);
}
static Json csvParse(const std::string& text,const Source& s) {
Json rows=Json::array(), row=Json::array(); std::string field;
bool quoted=false, closed=false, started=false; std::size_t nodes=1, bytes=128;
auto cell=[&]() {
bytes+=144+field.capacity(); if(++nodes>itemLimit || bytes>valueLimit) failLimit(s,"CSV storage limit");
row.push_back(std::move(field)); field.clear(); closed=false; started=false;
};
auto record=[&]() {
cell(); bytes+=144; if(++nodes>itemLimit || bytes>valueLimit) failLimit(s,"CSV storage limit");
rows.push_back(std::move(row)); row=Json::array();
};
for(std::size_t i=0;i<text.size();++i) {
char ch=text[i];
if(quoted) {
if(ch=='"') { if(i+1<text.size() && text[i+1]=='"') { field+='"'; ++i; } else { quoted=false; closed=true; } }
else field+=ch;
} else if(ch==',') cell();
else if(ch=='\r' || ch=='\n') {
if(ch=='\r') { if(i+1>=text.size() || text[i+1]!='\n') fail(s,"CSV bare CR outside quoted field"); ++i; }
record();
} else if(ch=='"') {
if(started || closed) fail(s,"CSV unexpected quote");
quoted=true; started=true;
} else { if(closed) fail(s,"CSV text after closing quote"); field+=ch; started=true; }
}
if(quoted) fail(s,"CSV unterminated quoted field");
if(started || closed || !row.empty() || !field.empty()) record();
return rows;
}
static std::string csvStringify(const Json& rows,const Source& s) {
if(!rows.is_array()) fail(s,"CSV requires JSON array of string rows");
std::string out;
for(const auto& row:rows) {
if(!row.is_array() || row.empty()) fail(s,"CSV rows must be nonempty arrays");
bool first=true;
for(const auto& cell:row) {
if(!cell.is_string()) fail(s,"CSV cells must be JSON strings");
if(!first) append(out,",",s);
first=false;
append(out,"\"",s);
for(char ch:cell.get_ref<const std::string&>()) { if(ch=='"') append(out,"\"",s); append(out,std::string(1,ch),s); }
append(out,"\"",s);
}
append(out,"\r\n",s);
}
return out;
}
struct vScript::Database {
sqlite3* db=nullptr;
bool internal=false, interrupted=false;
std::uint64_t ticks=0, tickLimit=1000;
std::chrono::steady_clock::time_point deadline;
~Database() { if(db) sqlite3_close_v2(db); }
static int authorize(void* context,int action,const char*,const char* second,const char*,const char*) noexcept {
auto& self=*static_cast<Database*>(context);
if(self.internal) return SQLITE_OK;
switch(action) {
case SQLITE_ATTACH: case SQLITE_DETACH: case SQLITE_PRAGMA:
case SQLITE_CREATE_VTABLE: case SQLITE_DROP_VTABLE:
case SQLITE_TRANSACTION: case SQLITE_SAVEPOINT: return SQLITE_DENY;
case SQLITE_FUNCTION:
if(second && (!sqlite3_stricmp(second,"load_extension") || !sqlite3_stricmp(second,"readfile") || !sqlite3_stricmp(second,"writefile"))) return SQLITE_DENY;
break;
default: break;
}
return SQLITE_OK;
}
static int progress(void* context) noexcept {
auto& self=*static_cast<Database*>(context);
if(++self.ticks>self.tickLimit || std::chrono::steady_clock::now()>self.deadline) { self.interrupted=true; return 1; }
return 0;
}
int control(const char* sql) {
internal=true; int rc=sqlite3_exec(db,sql,nullptr,nullptr,nullptr); internal=false; return rc;
}
};
bool vScript::worklog(Frame& frame,const CodeLine& c) {
const auto& a=c.arg; const auto& s=c.source; const auto& op=a[0].text;
if(op!="date" && op!="csv" && op!="db" && !(op=="file" && a[1].text=="create")) return false;
auto val=[&](std::size_t i) { return value(frame,a[i],s); };
auto put=[&](std::size_t i,const std::string& text) { assign(frame,a[i],text,s); };
const auto& sub=a[1].text;
if(op=="file") {
if(!options.allowFiles) fail(s,"file operations require --allow-files (not a sandbox)");
auto path=checkedPath(val(2),s), text=val(3);
if(text.size()>16*valueLimit-fileBytes) failLimit(s,"file write byte limit (16 MiB)");
fileBytes+=text.size();
int fd=::open(path.c_str(),O_WRONLY|O_CREAT|O_EXCL|O_CLOEXEC,0600);
if(fd<0) fail(s,"file create failed (destination must not exist)");
std::size_t offset=0;
while(offset<text.size()) {
auto n=::write(fd,text.data()+offset,text.size()-offset);
if(n<0 && errno==EINTR) continue;
if(n<=0) { ::close(fd); fail(s,"file create write failed"); }
offset+=static_cast<std::size_t>(n);
}
if(::close(fd)!=0) fail(s,"file create close failed");
} else if(op=="date") {
lookup(frame,a[2],s);
if(sub=="diff") {
auto end=integer(val(3),-62135647200LL,253402351200LL,s), start=integer(val(4),-62135647200LL,253402351200LL,s);
put(2,std::to_string(end-start));
} else {
auto text=val(3); int offset=dateOffset(val(4),s);
if(sub=="parse") put(2,std::to_string(dateParse(text,offset,s)));
else put(2,dateFormat(integer(text,-62135647200LL,253402351200LL,s),offset,s));
}
} else if(op=="csv") {
lookup(frame,a[2],s);
if(sub=="parse") putJson(frame,a[2],csvParse(val(3),s),s);
else put(2,csvStringify(jsonValue(frame,a[3],s),s));
} else {
if(!options.allowDb) fail(s,"database operations require --allow-db (not a sandbox)");
if(sub=="open") {
lookup(frame,a[2],s); auto path=checkedPath(val(3),s);
if(path.rfind("file:",0)==0) fail(s,"database URI filenames unavailable");
if(databases.size()>=8) failLimit(s,"database handle limit (8)");
auto db=std::make_shared<Database>();
int rc=sqlite3_open_v2(path.c_str(),&db->db,SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_PRIVATECACHE,nullptr);
if(rc!=SQLITE_OK) fail(s,"database open failed");
sqlite3_hard_heap_limit64(64*1024*1024);
sqlite3_enable_load_extension(db->db,0);
sqlite3_db_config(db->db,SQLITE_DBCONFIG_DEFENSIVE,1,nullptr);
sqlite3_db_config(db->db,SQLITE_DBCONFIG_TRUSTED_SCHEMA,0,nullptr);
sqlite3_busy_timeout(db->db,100);
sqlite3_limit(db->db,SQLITE_LIMIT_LENGTH,static_cast<int>(valueLimit));
sqlite3_limit(db->db,SQLITE_LIMIT_SQL_LENGTH,65536);
sqlite3_limit(db->db,SQLITE_LIMIT_COLUMN,128);
sqlite3_limit(db->db,SQLITE_LIMIT_EXPR_DEPTH,64);
sqlite3_limit(db->db,SQLITE_LIMIT_COMPOUND_SELECT,16);
sqlite3_limit(db->db,SQLITE_LIMIT_VARIABLE_NUMBER,256);
sqlite3_limit(db->db,SQLITE_LIMIT_VDBE_OP,25000);
sqlite3_limit(db->db,SQLITE_LIMIT_ATTACHED,0);
sqlite3_limit(db->db,SQLITE_LIMIT_TRIGGER_DEPTH,16);
sqlite3_limit(db->db,SQLITE_LIMIT_LIKE_PATTERN_LENGTH,4096);
if(db->control("PRAGMA temp_store=MEMORY")!=SQLITE_OK || db->control("PRAGMA cache_size=-2048")!=SQLITE_OK) fail(s,"database configuration failed");
sqlite3_set_authorizer(db->db,Database::authorize,db.get());
auto id=nextDatabase++; databases.emplace(id,db); put(2,std::to_string(id));
} else {
bool statement=sub=="exec" || sub=="query";
if(statement) lookup(frame,a[2],s);
auto id=static_cast<unsigned long>(integer(val(statement ? 3 : 2),1,100000000,s));
auto it=databases.find(id); if(it==databases.end()) fail(s,"invalid or closed database handle");
auto& d=*it->second;
if(sub=="close") { databases.erase(it); return true; }
d.ticks=0; d.tickLimit=std::min<std::uint64_t>(1000,(options.maxSteps-steps)/1000); d.interrupted=false; d.deadline=std::chrono::steady_clock::now()+std::chrono::seconds(2);
sqlite3_progress_handler(d.db,1000,Database::progress,&d);
try {
auto check=[&](int rc) { if(rc!=SQLITE_OK) fail(s,std::string("database error: ")+sqlite3_errmsg(d.db)); };
if(!statement) check(d.control(sub=="begin" ? "BEGIN" : sub=="commit" ? "COMMIT" : "ROLLBACK"));
else {
auto sql=val(4); const auto& params=jsonValue(frame,a[5],s);
if(sql.size()>65536) failLimit(s,"SQL byte limit (64 KiB)");
if(sql.find('\0')!=std::string::npos) fail(s,"SQL contains NUL");
if(!params.is_array()) fail(s,"database parameters require JSON array");
sqlite3_stmt* raw=nullptr; const char* tail=nullptr;
int rc=sqlite3_prepare_v2(d.db,sql.c_str(),static_cast<int>(sql.size()+1),&raw,&tail);
std::unique_ptr<sqlite3_stmt,decltype(&sqlite3_finalize)> stmt(raw,sqlite3_finalize);
check(rc); if(!raw) fail(s,"database requires one statement");
while(tail && *tail) {
sqlite3_stmt* extra=nullptr; const char* next=nullptr;
rc=sqlite3_prepare_v2(d.db,tail,-1,&extra,&next);
bool more=extra!=nullptr; if(extra) sqlite3_finalize(extra);
check(rc); if(more) fail(s,"database requires single statement");
if(next==tail) break;
tail=next;
}
if(params.size()!=static_cast<std::size_t>(sqlite3_bind_parameter_count(raw))) fail(s,"database parameter count mismatch");
for(std::size_t i=0;i<params.size();++i) {
const auto& p=params[i]; int index=static_cast<int>(i+1);
if(p.is_null()) rc=sqlite3_bind_null(raw,index);
else if(p.is_boolean()) rc=sqlite3_bind_int(raw,index,p.get<bool>() ? 1 : 0);
else if(p.is_number_integer()) rc=sqlite3_bind_int64(raw,index,p.get<sqlite3_int64>());
else if(p.is_number()) rc=sqlite3_bind_double(raw,index,p.get<double>());
else if(p.is_string()) { const auto& text=p.get_ref<const std::string&>(); rc=sqlite3_bind_text(raw,index,text.data(),static_cast<int>(text.size()),SQLITE_TRANSIENT); }
else fail(s,"database parameters must be scalar JSON");
check(rc);
}
int columns=sqlite3_column_count(raw);
if(sub=="query" && (!sqlite3_stmt_readonly(raw) || columns==0)) fail(s,"database query requires read-only rows");
if(sub=="exec" && columns) fail(s,"database exec cannot return rows");
std::vector<std::string> names;
for(int col=0;col<columns;++col) {
std::string name=sqlite3_column_name(raw,col);
if(std::find(names.begin(),names.end(),name)!=names.end()) fail(s,"duplicate database column name; use aliases");
names.push_back(std::move(name));
}
if(sub=="exec") check(d.control("SAVEPOINT vanilla_statement"));
Json rows=Json::array(); std::size_t bytes=128, nodes=1;
while((rc=sqlite3_step(raw))==SQLITE_ROW) {
if(rows.size()>=1024) failLimit(s,"database row limit (1024)");
Json row=Json::object(); nodes+=1+names.size();
if(nodes>itemLimit) failLimit(s,"database node limit");
for(int col=0;col<columns;++col) {
Json cell; int type=sqlite3_column_type(raw,col);
if(type==SQLITE_INTEGER) cell=sqlite3_column_int64(raw,col);
else if(type==SQLITE_FLOAT) cell=sqlite3_column_double(raw,col);
else if(type==SQLITE_TEXT) { const auto* text=sqlite3_column_text(raw,col); int size=sqlite3_column_bytes(raw,col); if(!text) fail(s,"database text allocation failed"); cell=std::string(reinterpret_cast<const char*>(text),static_cast<std::size_t>(size)); }
else if(type!=SQLITE_NULL) fail(s,"database BLOB results unsupported");
row[names[static_cast<std::size_t>(col)]]=std::move(cell);
}
bytes+=jsonSize(row,s)+sizeof(Json);
if(bytes>valueLimit) failLimit(s,"database result byte limit (1 MiB)");
rows.push_back(std::move(row));
}
if(rc!=SQLITE_DONE) check(rc);
stmt.reset();
if(sub=="exec") { auto changed=sqlite3_changes64(d.db); check(d.control("RELEASE vanilla_statement")); put(2,std::to_string(changed)); }
else {
if(bytes>16*valueLimit-databaseBytes) failLimit(s,"database cumulative result limit (16 MiB)");
databaseBytes+=bytes;
putJson(frame,a[2],std::move(rows),s);
}
}
sqlite3_progress_handler(d.db,0,nullptr,nullptr);
steps+=d.ticks*1000;
if(steps>options.maxSteps) failLimit(s,"step limit exceeded (database)");
} catch(...) {
sqlite3_progress_handler(d.db,0,nullptr,nullptr);
// Any failed DB operation aborts the active transaction, including
// statement SAVEPOINTs. Cleanup must not be interrupted again.
if(!sqlite3_get_autocommit(d.db)) d.control("ROLLBACK");
steps+=d.ticks*1000;
if(d.interrupted) failLimit(s,"database instruction/deadline limit");
throw;
}
}
}
return true;
}