// SQLite Storage Plugin
// Copyright (c) 2023 - 2026 Joseph Huckaby
// Released under the MIT License

// Requires the 'better-sqlite3' module

const fs = require('fs');
const Path = require('path');
const zlib = require('zlib');
const Class = require("pixl-class");
const Component = require("pixl-server/component");
const BetterSqlite3 = require('better-sqlite3');
const Tools = require("pixl-tools");
const Perf = require("pixl-perf");
const Cache = require("pixl-cache");
const async = require('async');
const noop = function() {};

module.exports = Class.create({
	
	__name: 'SQLite',
	__parent: Component,
	
	defaultConfig: {
		base_dir: '',
		filename: 'sqlite.db',
		keyPrefix: "",
		keyTemplate: ""
	},
	
	startup: function(callback) {
		// setup SQLite connection
		var self = this;
		this.logDebug(2, "Setting up SQLite", this.config.get() );
		this.setup(callback);
	},
	
	setup: function(callback) {
		// setup SQLite connection
		var self = this;
		var sql_config = this.config.get();
		
		this.keyPrefix = (sql_config.keyPrefix || '').replace(/^\//, '');
		if (this.keyPrefix && !this.keyPrefix.match(/\/$/)) this.keyPrefix += '/';
		
		this.keyTemplate = (sql_config.keyTemplate || '').replace(/^\//, '').replace(/\/$/, '');
		
		this.baseDir = sql_config.base_dir || process.cwd();
		this.commands = {
			get: 'SELECT value FROM items WHERE key = $key LIMIT 1',
			head: 'SELECT modified, length(value) FROM items WHERE key = $key LIMIT 1',
			put: 'INSERT INTO items VALUES($key, $value, $now) ON CONFLICT (key) DO UPDATE SET value = $value, modified = $now WHERE key = $key',
			delete: 'DELETE FROM items WHERE key = $key'
		};
		
		// create initial data dir if necessary
		try {
			Tools.mkdirp.sync( this.baseDir ); 
		}
		catch (e) {
			var msg = "FATAL ERROR: Base directory could not be created: " + this.baseDir + ": " + e;
			this.logError('sqlite', msg);
			throw new Error(msg);
		}
		
		// optional LRU cache
		this.cache = null;
		var cache_opts = this.config.get('cache');
		if (cache_opts && cache_opts.enabled) {
			this.logDebug(3, "Setting up LRU cache", cache_opts);
			this.cache = new Cache( Tools.copyHashRemoveKeys(cache_opts, { enabled: 1 }) );
			this.cache.on('expire', function(item, reason) {
				self.logDebug(9, "Expiring LRU cache object: " + item.key + " due to: " + reason, {
					key: item.key,
					reason: reason,
					totalCount: self.cache.count,
					totalBytes: self.cache.bytes
				});
			});
		}
		
		var db_file = Path.join( this.baseDir, sql_config.filename );
		this.logDebug(3, "Opening database file: " + db_file, sql_config.pragmas || {});
		
		try {
			// open sync connection
			this.db = new BetterSqlite3(db_file);
			
			// optionally set pragmas on the db
			if (sql_config.pragmas) {
				Object.keys(sql_config.pragmas).forEach(function(key) {
					var value = sql_config.pragmas[key];
					// value may be string (e.g. WAL) or number
					self.db.pragma(`${key} = ${value}`);
				});
			}
			
			// create our table if necessary
			this.db.prepare('CREATE TABLE IF NOT EXISTS items( key TEXT PRIMARY KEY, value BLOB, modified INTEGER )').run();
		}
		catch (err) {
			this.logError('sqlite', "FATAL ERROR: Database setup failed: " + err);
			return callback(err);
		}
		
		// behave asynchronously for API compatibility
		process.nextTick(function() {
			self.logDebug(3, "Setup complete");
			callback();
		});
	},
	
	prepKey: function(key) {
		// prepare key based on config
		var md5 = Tools.digestHex(key, 'md5');
		
		if (this.keyPrefix) {
			key = this.keyPrefix + key;
		}
		
		if (this.keyTemplate) {
			var idx = 0;
			var temp = this.keyTemplate.replace( /\#/g, function() {
				return md5.substr(idx++, 1);
			} );
			key = Tools.substitute( temp, { key: key, md5: md5 } );
		}
		
		return key;
	},
	
	put: function(key, value, callback) {
		// store key+value in SQLite
		var self = this;
		var now = Tools.timeNow(true);
		key = this.prepKey(key);
		
		var is_binary = this.storage.isBinaryKey(key);
		
		if (is_binary) {
			this.logDebug(9, "Storing SQLite Binary Object: " + key, '' + value.length + ' bytes');
		}
		else {
			this.logDebug(9, "Storing SQLite JSON Object: " + key, this.debugLevel(10) ? value : null);
			value = Buffer.from( JSON.stringify( value ) );
		}
		
		process.nextTick(function() {
			try {
				self.db.prepare(self.commands.put).run({ key: key, value: value, now: now });
			}
			catch (err) {
				err.message = "Failed to store object: " + key + ": " + err;
				self.logError('sqlite', '' + err);
				if (callback) callback(err);
				return;
			}
			
			self.logDebug(9, "Store complete: " + key);
			// possibly cache in LRU
			if (self.cache && !is_binary) {
				self.cache.set( key, value, { date: Tools.timeNow(true) } );
			}
			if (callback) callback(null);
		});
	},
	
	putStream: function(key, inp, callback) {
		// store key+value in SQLite using read stream
		var self = this;
		
		// The SQLite API has no stream support.
		// So, we have to do this the RAM-hard way...
		
		var chunks = [];
		inp.on('data', function(chunk) {
			chunks.push( chunk );
		} );
		inp.on('end', function() {
			var buf = Buffer.concat(chunks);
			self.put( key, buf, callback );
		} );
	},
	
	head: function(key, callback) {
		// head sqlite item given key
		var self = this;
		key = this.prepKey(key);
		
		// check cache first
		if (this.cache && this.cache.has(key)) {
			var item = this.cache.getMeta(key);
			
			process.nextTick( function() {
				self.logDebug(9, "Cached head complete: " + key);
				callback( null, {
					mod: item.date,
					len: item.value.length
				} );
			} );
			return;
		} // cache
		
		process.nextTick(function() {
			var row = null;
			try {
				row = self.db.prepare(self.commands.head).get({ key: key });
			}
			catch (err) {
				err.message = "Failed to head key: " + key + ": " + err;
				self.logError('sqlite', '' + err);
				return callback(err);
			}
			
			if (!row) {
				var err = new Error("Failed to head key: " + key + ": Not found");
				err.code = "NoSuchKey";
				return callback(err, null);
			}
			callback(null, { mod: row.modified, len: row['length(value)'] });
		});
	},
	
	get: function(key, callback) {
		// fetch SQLite value given key
		var self = this;
		key = this.prepKey(key);
		
		var is_binary = this.storage.isBinaryKey(key);
		
		// check cache first
		if (this.cache && !is_binary && this.cache.has(key)) {
			var data = this.cache.get(key);
			
			process.nextTick( function() {
				try { data = JSON.parse( data.toString() ); }
				catch (e) {
					self.logError('file', "Failed to parse JSON record: " + key + ": " + e);
					callback( e, null );
					return;
				}
				self.logDebug(9, "Cached JSON fetch complete: " + key, self.debugLevel(10) ? data : null);
				
				callback( null, data );
			} );
			return;
		} // cache
		
		this.logDebug(9, "Fetching SQLite Object: " + key);
		
		process.nextTick(function() {
			var row = null;
			try {
				row = self.db.prepare(self.commands.get).get({ key: key });
			}
			catch (err) {
				err.message = "Failed to fetch key: " + key + ": " + err;
				self.logError('sqlite', '' + err);
				callback(err);
				return;
			}
			
			if (!row) {
				var err = new Error("Failed to fetch key: " + key + ": Not found");
				err.code = "NoSuchKey";
				return callback(err, null);
			}
			if (is_binary) {
				self.logDebug(9, "Binary fetch complete: " + key, '' + row.value.length + ' bytes');
				return callback(null, row.value);
			}
			if (self.cache) {
				self.cache.set( key, row.value, { date: Tools.timeNow(true) } );
			}
			var json = null;
			try { json = JSON.parse( row.value.toString() ); }
			catch (err) {
				self.logError('sqlite', "Failed to parse JSON record: " + key + ": " + err);
				return callback(err, null);
			}
			self.logDebug(9, "JSON fetch complete: " + key, self.debugLevel(10) ? json : null);
			callback(null, json);
		});
	},
	
	getBuffer: function(key, callback) {
		// fetch SQLite buffer given key
		var self = this;
		key = this.prepKey(key);
		
		this.logDebug(9, "Fetching SQLite Object: " + key);
		
		process.nextTick(function() {
			var row = null;
			try {
				row = self.db.prepare(self.commands.get).get({ key: key });
			}
			catch (err) {
				err.message = "Failed to fetch key: " + key + ": " + err;
				self.logError('sqlite', '' + err);
				callback(err);
				return;
			}
			
			if (!row) {
				var err = new Error("Failed to fetch key: " + key + ": Not found");
				err.code = "NoSuchKey";
				return callback(err, null);
			}
			self.logDebug(9, "Binary fetch complete: " + key, '' + row.value.length + ' bytes');
			callback(null, row.value);
		});
	},
	
	getStream: function(key, callback) {
		// get readable stream to record value given key
		var self = this;
		
		// The SQLite API has no stream support.
		// So, we have to do this the RAM-hard way...
		
		this.get( key, function(err, buf) {
			if (err) {
				// an actual error
				err.message = "Failed to fetch key: " + key + ": " + err;
				self.logError('sqlite', '' + err);
				return callback(err);
			}
			else if (!buf) {
				// record not found
				var err = new Error("Failed to fetch key: " + key + ": Not found");
				err.code = "NoSuchKey";
				return callback( err, null );
			}
			
			var stream = new BufferStream(buf);
			callback(null, stream, { mod: 1, len: buf.length });
		} );
	},
	
	getStreamRange: function(key, start, end, callback) {
		// get readable stream to record value given key and range
		var self = this;
		
		// The SQLite API has no stream support.
		// So, we have to do this the RAM-hard way...
		
		this.get( key, function(err, buf) {
			if (err) {
				// an actual error
				err.message = "Failed to fetch key: " + key + ": " + err;
				self.logError('sqlite', '' + err);
				return callback(err);
			}
			else if (!buf) {
				// record not found
				var err = new Error("Failed to fetch key: " + key + ": Not found");
				err.code = "NoSuchKey";
				return callback( err, null );
			}
			
			// validate byte range, now that we have the head info
			if (isNaN(start) && !isNaN(end)) {
				start = buf.length - end;
				end = buf.length ? buf.length - 1 : 0;
			} 
			else if (!isNaN(start) && isNaN(end)) {
				end = buf.length ? buf.length - 1 : 0;
			}
			if (isNaN(start) || isNaN(end) || (start < 0) || (start >= buf.length) || (end < start) || (end >= buf.length)) {
				callback( new Error("Invalid byte range (" + start + '-' + end + ") for key: " + key + " (len: " + buf.length + ")"), null );
				return;
			}
			
			var range = buf.slice(start, end + 1);
			var stream = new BufferStream(range);
			callback(null, stream, { mod: 1, len: buf.length });
		} );
	},
	
	delete: function(key, callback) {
		// delete SQLite key given key
		var self = this;
		key = this.prepKey(key);
		
		this.logDebug(9, "Deleting SQLite Object: " + key);
		
		process.nextTick(function() {
			var info = null;
			try {
				info = self.db.prepare(self.commands.delete).run({ key: key });
			}
			catch (err) {
				self.logError('sqlite', "Failed to delete object: " + key + ": " + err);
				callback(err);
				return;
			}
			
			if (!info.changes) {
				var err = new Error("Failed to delete object: " + key + ": Not found");
				err.code = "NoSuchKey";
				return callback(err);
			}
			self.logDebug(9, "Delete complete: " + key);
			if (self.cache && self.cache.has(key)) {
				self.cache.delete(key);
			}
			callback();
		});
	},
	
	runMaintenance: function(callback) {
		// run daily maintenance
		// config.backups: { enabled, dir, filename, compress?, keep? }
		var self = this;
		this.logDebug(3, "Running SQLite maintenance");
		
		var backups = this.config.get('backups') || null;
		if (!backups || !backups.enabled || !backups.dir || !backups.filename) return callback();
		
		// here be dragons: try to wait for a moment BETWEEN transaction commits, to minimize risk of an "inconsistent" backup
		if (this.storage.transactions && Tools.findObject(Object.values(this.storage.transactions), { committing: true })) {
			// wait for next commitEnd event and try again
			this.logDebug(9, "Maintenance: Transaction commit(s) are in progress, waiting until next commitEnd event...");
			
			this.storage.once('commitEnd', function(trans) {
				self.logDebug(9, "Maintenance: Transaction commit " + trans.id + " (" + trans.path + ") ended, retrying now...");
				self.runMaintenance(callback);
			});
			
			return;
		}
		
		// track perf for logging
		var perf = new Perf();
		perf.begin();
		perf.begin('prep');
		
		// allow dir & filename to have date/time placeholders, e.g. `backup-[yyyy]-[mm]-[dd].db`
		var dargs = Tools.getDateArgs( Tools.timeNow() );
		var file = Tools.sub( Path.join(backups.dir, backups.filename), dargs, true );
		if (!file) {
			this.logError('backup', "Failed to expand placeholders on backup file: " + backups.filename);
			return callback();
		}
		
		// strip .gz prefix, just in case the user added it
		file = file.replace(/\.gz$/i, '');
		
		// auto-create parent dirs as needed
		if (!fs.existsSync(backups.dir)) {
			try { Tools.mkdirp.sync(backups.dir); }
			catch (err) {
				this.logError('backup', "Failed to create backup directory: " + backups.dir);
				return callback();
			}
		}
		
		// here we go
		if (fs.existsSync(file)) fs.unlinkSync(file);
		this.logDebug(6, "Performing database backup to: " + file);
		perf.end('prep');
		
		var finish = function() {
			// all done
			perf.end();
			self.logDebug(6, "Maintenance complete", perf.metrics());
			callback();
		};
		
		var keep = function() {
			// delete oldest backups if over N on disk
			var files = Tools.findFilesSync( backups.dir, { stats: true } );
			if (files.length <= backups.keep) return finish();
			perf.begin('keep');
			
			self.logDebug(6, "There are " + files.length + " backup files in " + backups.dir + ", keeping the latest " + backups.keep);
			
			// sort mtime descending so the newest are at the start of the array, then splice those off, leaving only the oldest to delete
			Tools.sortBy( files, 'mtime', { type: 'number', dir: -1 } );
			files.splice( 0, backups.keep );
			
			files.forEach( function(file) {
				self.logDebug(7, "Deleting old backup file: " + file.path, file);
				try { fs.unlinkSync(file.path); } catch (err) {
					self.logError('backup', "Failed to delete backup file: " + file.path + ": " + err);
				}
			} );
			
			perf.end('keep');
			finish();
		}; // keep
		
		var compress = function() {
			// compress backup and delete original
			var gz_file = file + '.gz';
			perf.begin('compress');
			
			if (fs.existsSync(gz_file)) fs.unlinkSync(gz_file);
			self.logDebug(6, "Compressing backup to: " + gz_file);
			
			var gzip = zlib.createGzip();
			var inp = fs.createReadStream( file );
			var outp = fs.createWriteStream( gz_file );
			
			var handleError = function(err) {
				self.logError('backup', "Failed to compress backup: " + gz_file + ": " + err);
				fs.unlink(gz_file, noop);
				
				perf.end('compress');
				if (backups.keep) keep();
				else finish();
			};
			
			gzip.on('error', handleError);
			inp.on('error', handleError);
			outp.on('error', handleError);
			
			inp.pipe(gzip).pipe(outp).on('finish', function() {
				self.logDebug(6, "Backup compression complete: " + gz_file);
				fs.unlink(file, noop);
				
				perf.end('compress');
				if (backups.keep) keep();
				else finish();
			} );
		}; // compress
		
		// Perform the backup using better-sqlite3's async backup API
		// As long as WAL mode is enabled, this should only acquire a shared lock
		perf.begin('backup');
		var db = this.db;
		db.backup(file, {
			progress: function(info) {
				// transfer pages per tick (tuneable)
				return 100;
			}
		})
		.then(function() {
			perf.end('backup');
			self.logDebug(6, "SQLite backup operation completed: " + file);
			// now compress or keep or done
			if (backups.compress) compress();
			else if (backups.keep) keep();
			else finish();
		})
		.catch(function(err) {
			perf.end('backup');
			self.logError('backup', "SQLite backup operation failed.  Please check permissions and available disk space. " + err);
			fs.unlink(file, noop);
			finish();
		});
	},
	
	shutdown: function(callback) {
		// shutdown storage
		var self = this;
		this.logDebug(2, "Shutting down SQLite");
		
		if (this.db) {
			try { this.db.close(); }
			catch (err) { this.logError('sqlite', "Failed to shutdown database cleanly: " + err); }
			this.db = null;
			this.commands = null;
		}
		process.nextTick(function() {
			self.logDebug(3, "Shutdown complete");
			callback();
		});
	}
	
});

// Modified the following snippet from node-streamifier:
// Copyright (c) 2014 Gabriel Llamas, MIT Licensed

var util = require('util');
var stream = require('stream');

var BufferStream = function (object, options) {
	if (object instanceof Buffer || typeof object === 'string') {
		options = options || {};
		stream.Readable.call(this, {
			highWaterMark: options.highWaterMark,
			encoding: options.encoding
		});
	} else {
		stream.Readable.call(this, { objectMode: true });
	}
	this._object = object;
};

util.inherits(BufferStream, stream.Readable);

BufferStream.prototype._read = function () {
	this.push(this._object);
	this._object = null;
};
