ziver/hal
1
1
Fork 0
Code Pull requests Releases Activity

Initial implementation of plugin DB upgrade support

Browse source
This commit is contained in:
Ziver Koc 2021-07-11 23:00:47 +02:00
parent aea0d9bab1
commit 6a81f5de16
11 changed files with 295 additions and 106 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
hal-core/resource/resource/hal-default.db → hal-core/resource/resource/hal-core-reference.db
View file

Binary file not shown.

96
hal-core/src/se/hal/HalContext.java
View file

@ -42,7 +42,7 @@ public class HalContext {
public static final String RESOURCE_WEB_ROOT = HalContext.RESOURCE_ROOT + "/resource/web";
private static final String CONF_FILE = "hal.conf";
private static final String DB_FILE = "hal.db";
static final String DB_FILE = "hal.db";
private static final String DEFAULT_DB_FILE = HalContext.RESOURCE_ROOT + "/resource/hal-default.db";
// Variables
@ -85,102 +85,8 @@ public class HalContext {
dbConf = db.exec("SELECT * FROM conf", new PropertiesSQLResult());
}
// Upgrade DB needed?
DBConnection referenceDB = new DBConnection(DBConnection.DBMS.SQLite, DEFAULT_DB_FILE);
Properties defaultDBConf =
referenceDB.exec("SELECT * FROM conf", new PropertiesSQLResult());
// Check DB version
final int defaultDBVersion = Integer.parseInt(defaultDBConf.getProperty(PROPERTY_DB_VERSION));
final int dbVersion = (dbConf.getProperty(PROPERTY_DB_VERSION) != null ?
Integer.parseInt(dbConf.getProperty(PROPERTY_DB_VERSION)) :
-1);
logger.info("DB version: "+ dbVersion);
if (defaultDBVersion > dbVersion ) {
logger.info("Starting DB upgrade from v" + dbVersion + " to v" + defaultDBVersion + "...");
if (dbFile != null){
File backupDB = FileUtil.getNextFile(dbFile);
logger.fine("Backing up DB to: "+ backupDB);
FileUtil.copy(dbFile, backupDB);
}
logger.fine(String.format("Upgrading DB (from: v%s, to: v%s)...", dbVersion, defaultDBVersion));
final DBUpgradeHandler handler = new DBUpgradeHandler(referenceDB);
handler.addIgnoredTable("db_version_history");
handler.addIgnoredTable("sqlite_sequence"); // sqlite internal
handler.setTargetDB(db);
logger.fine("Performing pre-upgrade activities");
// Read upgrade path preferences from the reference database
referenceDB.exec("SELECT * FROM db_version_history"
+ " WHERE db_version <= " + defaultDBVersion
+ " AND db_version > " + dbVersion,
new SQLResultHandler<Object>() {
@Override
public Object handleQueryResult(Statement stmt, ResultSet result) throws SQLException {
while (result.next()){
if (result.getBoolean("force_upgrade")){
logger.fine("Forced upgrade enabled");
handler.setForcedDBUpgrade(true); //set to true if any of the intermediate db version requires it.
}
}
return null;
}
});
handler.upgrade();
logger.fine("Performing post-upgrade activities");
// Read upgrade path preferences from the reference database
referenceDB.exec("SELECT * FROM db_version_history"
+ " WHERE db_version <= " + defaultDBVersion
+ " AND db_version > " + dbVersion,
new SQLResultHandler<Object>() {
@Override
public Object handleQueryResult(Statement stmt, ResultSet result) throws SQLException {
boolean clearExternalAggrData = false;
boolean clearInternalAggrData = false;
while (result.next()){
if (result.getBoolean("clear_external_aggr_data"))
clearExternalAggrData = true;
if (result.getBoolean("clear_internal_aggr_data"))
clearInternalAggrData = true;
}
if (clearExternalAggrData){
logger.fine("Clearing external aggregate data");
db.exec("DELETE FROM sensor_data_aggr WHERE sensor_id = "
+ "(SELECT sensor.id FROM user, sensor WHERE user.external == 1 AND sensor.user_id = user.id)");
}
if (clearInternalAggrData){
logger.fine("Clearing local aggregate data");
db.exec("DELETE FROM sensor_data_aggr WHERE sensor_id IN "
+ "(SELECT sensor.id FROM user, sensor WHERE user.external == 0 AND sensor.user_id = user.id)");
//update all internal sensors aggregation version to indicate for peers that they need to re-sync all data
db.exec("UPDATE sensor SET aggr_version = (aggr_version+1) WHERE id = "
+ "(SELECT sensor.id FROM user, sensor WHERE user.external == 0 AND sensor.user_id = user.id)");
}
return null;
}
});
// Check if there is a local user
User localUser = User.getLocalUser(db);
if (localUser == null){
logger.fine("Creating local user.");
localUser = new User();
localUser.setExternal(false);
localUser.save(db);
}
logger.info("DB upgrade done");
setProperty(PROPERTY_DB_VERSION, defaultDBConf.getProperty(PROPERTY_DB_VERSION));
}
referenceDB.close();
} catch (Exception e){
throw new RuntimeException(e);
}

44
hal-core/src/se/hal/HalCoreDatabaseUpgrade.java Normal file
View file

@ -0,0 +1,44 @@
package se.hal;
import se.hal.intf.HalDatabaseUpgrade;
import zutil.db.DBConnection;
import zutil.log.LogUtil;
import java.sql.SQLException;
import java.util.logging.Logger;
/**
* The DB upgrade class for Hal-Core
*/
public class HalCoreDatabaseUpgrade extends HalDatabaseUpgrade {
private static final Logger logger = LogUtil.getLogger();
private static final int REFERENCE_DB_VERSION = 16;
private static final String REFERENCE_DB_PATH = "resource/hal-core-reference.db";
private static final int CLEAR_INTERNAL_AGGR_DATA_DB_VERSION = 11;
private static final int CLEAR_EXTERNAL_AGGR_DATA_DB_VERSION = 0;
public HalCoreDatabaseUpgrade() {
super(REFERENCE_DB_VERSION, REFERENCE_DB_PATH);
}
@Override
public void postDatabaseUpgrade(DBConnection db, int fromDBVersion, int toDBVersion) throws SQLException {
if (fromDBVersion <= CLEAR_EXTERNAL_AGGR_DATA_DB_VERSION){
logger.fine("Clearing external aggregate data.");
db.exec("DELETE FROM sensor_data_aggr WHERE sensor_id = "
+ "(SELECT sensor.id FROM user, sensor WHERE user.external == 1 AND sensor.user_id = user.id)");
}
if (fromDBVersion <= CLEAR_INTERNAL_AGGR_DATA_DB_VERSION){
logger.fine("Clearing local aggregate data.");
db.exec("DELETE FROM sensor_data_aggr WHERE sensor_id IN "
+ "(SELECT sensor.id FROM user, sensor WHERE user.external == 0 AND sensor.user_id = user.id)");
// Update all internal sensors aggregation version to indicate for peers that they need to re-sync all data
db.exec("UPDATE sensor SET aggr_version = (aggr_version+1) WHERE id = "
+ "(SELECT sensor.id FROM user, sensor WHERE user.external == 0 AND sensor.user_id = user.id)");
}
}
}

160
hal-core/src/se/hal/HalDatabaseUpgradeManager.java Normal file
View file

@ -0,0 +1,160 @@
package se.hal;
import se.hal.intf.HalDatabaseUpgrade;
import se.hal.struct.User;
import zutil.db.DBConnection;
import zutil.db.DBUpgradeHandler;
import zutil.db.handler.PropertiesSQLResult;
import zutil.io.file.FileUtil;
import zutil.log.LogUtil;
import zutil.plugin.PluginManager;
import java.io.File;
import java.sql.PreparedStatement;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.Properties;
import java.util.Queue;
import java.util.logging.Logger;
/**
* A manager class handling all upgrades for the main Hal DB and any plugin changes.
*/
public class HalDatabaseUpgradeManager {
private static final Logger logger = LogUtil.getLogger();
private static Queue<HalDatabaseUpgrade> upgradeQueue;
/**
* Method will read in all HalDatabaseUpgrade plugins and populate the upgrade queue.
* Node, method will only read in plugins on the first call, any subsequent calls will be ignored.
*
* @param pluginManager
*/
public static void initialize(PluginManager pluginManager) {
if (upgradeQueue != null)
return;
upgradeQueue = new LinkedList<>();
for (Iterator<HalDatabaseUpgrade> it = pluginManager.getSingletonIterator(HalDatabaseUpgrade.class); it.hasNext(); ) {
HalDatabaseUpgrade dbUpgrade = it.next();
upgradeQueue.add(dbUpgrade);
}
}
/**
* Method will execute all queued database upgrades.
*/
public static void upgrade() {
if (upgradeQueue == null)
return;
while (!upgradeQueue.isEmpty()) {
upgrade(upgradeQueue.poll());
}
}
private synchronized static void upgrade(HalDatabaseUpgrade dbUpgrade) {
DBConnection mainDB = null;
DBConnection referenceDB = null;
String referenceDBPath = dbUpgrade.getReferenceDBPath();
try {
if (FileUtil.find(referenceDBPath) == null)
throw new IllegalArgumentException("Unable to find default DB: " + referenceDBPath);
// Init DB
File dbFile = FileUtil.find(HalContext.DB_FILE);
mainDB = new DBConnection(DBConnection.DBMS.SQLite, HalContext.DB_FILE);
Properties dbConf =
mainDB.exec("SELECT * FROM conf", new PropertiesSQLResult());
if (dbFile == null) {
logger.info("No database file found, creating new DB...");
}
// Upgrade DB needed?
referenceDB = new DBConnection(DBConnection.DBMS.SQLite, referenceDBPath);
String mainDBVersionProperty = dbUpgrade.getClass().getSimpleName() + ".db_version";
// Check DB version
final int referenceDBVersion = dbUpgrade.getReferenceDBVersion();
final int mainDBVersion = (dbConf.getProperty(mainDBVersionProperty) != null ?
Integer.parseInt(dbConf.getProperty(mainDBVersionProperty)) :
-1);
logger.info("DB version: " + mainDBVersion);
if (referenceDBVersion > mainDBVersion) {
// ----------------------------------------
// Prepare upgrade
// ----------------------------------------
logger.info("Starting DB upgrade from v" + mainDBVersion + " to v" + referenceDBVersion + "...");
if (dbFile != null){
File backupDB = FileUtil.getNextFile(dbFile);
logger.fine("Backing up DB to: "+ backupDB);
FileUtil.copy(dbFile, backupDB);
}
final DBUpgradeHandler handler = new DBUpgradeHandler(referenceDB);
handler.addIgnoredTable("sqlite_sequence"); // sqlite internal
handler.setTargetDB(mainDB);
logger.fine("Performing pre-upgrade activities");
/*if (doForceUpgrade) {
logger.fine("Forced upgrade enabled.");
handler.setForcedDBUpgrade(true); // set to true if any of the intermediate db version requires it.
}*/
dbUpgrade.preDatabaseUpgrade(mainDB, mainDBVersion, referenceDBVersion);
// ----------------------------------------
// Upgrade
// ----------------------------------------
handler.upgrade();
// ----------------------------------------
// Post-upgrade
// ----------------------------------------
logger.fine("Performing post-upgrade activities.");
// Check if there is a local user
User localUser = User.getLocalUser(mainDB);
if (localUser == null){
logger.fine("Creating local user.");
localUser = new User();
localUser.setExternal(false);
localUser.save(mainDB);
}
dbUpgrade.postDatabaseUpgrade(mainDB, mainDBVersion, referenceDBVersion);
// Update DB version
PreparedStatement stmt = mainDB.getPreparedStatement("REPLACE INTO conf (key, value) VALUES (?, ?)");
stmt.setString(1, mainDBVersionProperty);
stmt.setInt(2, referenceDBVersion);
DBConnection.exec(stmt);
logger.info("DB upgrade done.");
} else {
logger.info("No DB upgrade needed");
}
} catch (Exception e){
throw new RuntimeException(e);
} finally {
if (mainDB != null)
mainDB.close();
if (referenceDB != null)
referenceDB.close();
}
}
}

19
hal-core/src/se/hal/HalServer.java
View file

@ -1,10 +1,7 @@
package se.hal;
import se.hal.intf.HalAbstractControllerManager;
import se.hal.intf.HalDaemon;
import se.hal.intf.HalJsonPage;
import se.hal.intf.HalWebPage;
import se.hal.intf.*;
import se.hal.page.HalAlertManager;
import se.hal.struct.PluginConfig;
import zutil.db.DBConnection;
@ -52,18 +49,22 @@ public class HalServer {
// init logging
LogUtil.readConfiguration("logging.properties");
// init DB and other configurations
HalContext.initialize();
logger.info("Working directory: " + FileUtil.find(".").getAbsolutePath());
// init variables
pluginManager = new PluginManager();
daemonExecutor = Executors.newScheduledThreadPool(1); // We set only one thread for easier troubleshooting
http = new HttpServer(HalContext.getIntegerProperty(HalContext.PROPERTY_HTTP_PORT));
// Upgrade database
HalDatabaseUpgradeManager.initialize(pluginManager);
HalDatabaseUpgradeManager.upgrade();
// init DB and other configurations
HalContext.initialize();
DBConnection db = HalContext.getDB();
logger.info("Working directory: " + FileUtil.find(".").getAbsolutePath());
// ------------------------------------
// Initialize Plugins
// ------------------------------------

53
hal-core/src/se/hal/intf/HalDatabaseUpgrade.java Normal file
View file

@ -0,0 +1,53 @@
package se.hal.intf;
import zutil.db.DBConnection;
import java.sql.SQLException;
/**
* A plugin interface for custom plugin DB changes.
*/
public abstract class HalDatabaseUpgrade {
private int referenceDBVersion;
private String referenceDBPath;
public HalDatabaseUpgrade(int referenceDBVersion, String referenceDBPath) {
this.referenceDBVersion = referenceDBVersion;
this.referenceDBPath = referenceDBPath;
}
/**
* @return the reference DB version which will be used as a to state during the upgrade process.
*/
public int getReferenceDBVersion() {
return referenceDBVersion;
}
/**
* @return the path to the reference DB file that represents the target structure of the DB.
*/
public String getReferenceDBPath() {
return referenceDBPath;
}
/**
* Method will be called just before the DB upgrade is performed.
*
* @param db Connection to the DB that will be upgraded.
* @param fromDBVersion The current version of the to be upgraded DB.
* @param toDBVersion The target version that the DB will be upgraded to.
*/
public void preDatabaseUpgrade(DBConnection db, int fromDBVersion, int toDBVersion) throws SQLException {}
/**
* Method will be called after the DB has been upgraded.
*
* @param db Connection to the upgraded DB.
* @param fromDBVersion The before upgrade version of the DB.
* @param toDBVersion The target version of the DB.
*/
public void postDatabaseUpgrade(DBConnection db, int fromDBVersion, int toDBVersion) throws SQLException {}
}

2
hal-core/src/se/hal/plugin.json
View file

@ -3,6 +3,8 @@
"name": "Hal-Core",
"description": "Plugin contains core logic for running Hal.",
"interfaces": [
{"se.hal.intf.HalDatabaseUpgrade": "se.hal.HalCoreDatabaseUpgrade"},
{"se.hal.intf.HalAbstractControllerManager": "se.hal.EventControllerManager"},
{"se.hal.intf.HalAbstractControllerManager": "se.hal.SensorControllerManager"},

1
hal-core/test/se/hal/daemon/SensorDataAggregationDaemonTest.java
View file

@ -32,7 +32,6 @@ public class SensorDataAggregationDaemonTest {
System.out.println("Upgrading in-memory database to latest version");
DBConnection referenceDB = new DBConnection(DBConnection.DBMS.SQLite, DEFAULT_DB_FILE);
final DBUpgradeHandler handler = new DBUpgradeHandler(referenceDB);
handler.addIgnoredTable("db_version_history");
handler.addIgnoredTable("sqlite_sequence"); //sqlite internal
handler.setTargetDB(db);
handler.upgrade();

BIN
plugins/hal-zigbee/resource/resource/hal-zigbee-default.db → plugins/hal-zigbee/resource/resource/hal-zigbee-reference.db
View file

Binary file not shown.

22
plugins/hal-zigbee/src/se/hal/plugin/zigbee/ZigbeeHalDatabaseUpgrade.java Normal file
View file

@ -0,0 +1,22 @@
package se.hal.plugin.zigbee;
import se.hal.intf.HalDatabaseUpgrade;
import zutil.db.DBConnection;
import zutil.log.LogUtil;
import java.sql.SQLException;
import java.util.logging.Logger;
/**
* The DB upgrade class for Hal-Core
*/
public class ZigbeeHalDatabaseUpgrade extends HalDatabaseUpgrade {
private static final int REFERENCE_DB_VERSION = 1;
private static final String REFERENCE_DB_PATH = "resource/hal-zigbee-reference.db";
public ZigbeeHalDatabaseUpgrade() {
super(REFERENCE_DB_VERSION, REFERENCE_DB_PATH);
}
}

4
plugins/hal-zigbee/src/se/hal/plugin/zigbee/plugin.json
View file

@ -1,8 +1,10 @@
{
"version": 0.1,
"name": "Hal-Zigbee",
"description": "A Zigbee plugin for directly connecting to a CC2531 device over serial port.",
"description": "A Zigbee plugin for directly connecting to a controller device over serial port.",
"interfaces": [
{"se.hal.intf.HalDatabaseUpgrade": "se.hal.plugin.zigbee.ZigbeeHalDatabaseUpgrade"},
{"se.hal.intf.HalEventConfig": "se.hal.plugin.zigbee.device.ZigbeeOnOffConfig"},
{"se.hal.intf.HalSensorConfig": "se.hal.plugin.zigbee.device.ZigbeeHumidityConfig"},
{"se.hal.intf.HalSensorConfig": "se.hal.plugin.zigbee.device.ZigbeePressureConfig"},