From 5f97014bdc6344f8dc1f0165862358f14a4d8140 Mon Sep 17 00:00:00 2001 From: Alessio Date: Mon, 22 Nov 2021 16:55:27 -0800 Subject: [PATCH] Add database versioning --- persistence/profile.go | 8 +++- persistence/profile_test.go | 9 ++++ persistence/schema.sql | 4 ++ persistence/versions.go | 90 ++++++++++++++++++++++++++++++++++++ persistence/versions_test.go | 33 +++++++++++++ 5 files changed, 142 insertions(+), 2 deletions(-) create mode 100644 persistence/versions.go create mode 100644 persistence/versions_test.go diff --git a/persistence/profile.go b/persistence/profile.go index 801bdaf..65f9c1f 100644 --- a/persistence/profile.go +++ b/persistence/profile.go @@ -82,6 +82,7 @@ func NewProfile(target_dir string) (Profile, error) { if err != nil { return Profile{}, err } + InitializeDatabaseVersion(db) // Create `users.txt` fmt.Printf("Creating............. %s\n", user_list_file) @@ -183,10 +184,13 @@ func LoadProfile(profile_dir string) (Profile, error) { return Profile{}, err } - return Profile{ + ret := Profile{ ProfileDir: profile_dir, UsersList: users_list, Settings: settings, DB: db, - }, nil + } + err = ret.check_and_update_version() + + return ret, err } diff --git a/persistence/profile_test.go b/persistence/profile_test.go index 5d6a9f6..27ee0ab 100644 --- a/persistence/profile_test.go +++ b/persistence/profile_test.go @@ -105,6 +105,15 @@ func TestNewProfile(t *testing.T) { t.Fatalf("Expected `%s` to be a %s, but got %s [%s]", v.filename, isdir_map(v.isDir), contents[i].Name(), isdir_map(contents[i].IsDir())) } } + + // Check database version is initialized + version, err := profile.GetDatabaseVersion() + if err != nil { + panic(err) + } + if version != 0 { + t.Errorf("Expected database version %d, but got %d", 0, version) + } } diff --git a/persistence/schema.sql b/persistence/schema.sql index 5d84fc7..814e398 100644 --- a/persistence/schema.sql +++ b/persistence/schema.sql @@ -113,3 +113,7 @@ create table hashtags (rowid integer primary key, unique (tweet_id, text) foreign key(tweet_id) references tweets(id) ); + +create table database_version(rowid integer primary key, + version_number integer not null unique +); diff --git a/persistence/versions.go b/persistence/versions.go new file mode 100644 index 0000000..1e61744 --- /dev/null +++ b/persistence/versions.go @@ -0,0 +1,90 @@ +package persistence + +import ( + "fmt" + "database/sql" +) + + +const ENGINE_DATABASE_VERSION = 0 + + +type VersionMismatchError struct { + EngineVersion int + DatabaseVersion int +} +func (e VersionMismatchError) Error() string { + return fmt.Sprintf( +`This profile was created with database schema version %d, which is newer than this application's database schema version, %d. +Please upgrade this application to a newer version to use this profile. Or downgrade the profile's schema version, somehow.`, + e.DatabaseVersion, e.EngineVersion, + ) +} + + +/** + * The Nth entry is the migration that moves you from version N to version N+1 + */ +var MIGRATIONS = []string{ + +} + +/** + * This should only get called on a newly created Profile. + * Subsequent updates should change the number, not insert a new row. + */ +func InitializeDatabaseVersion(db *sql.DB) { + _, err := db.Exec("insert into database_version (version_number) values (?)", ENGINE_DATABASE_VERSION) + if err != nil { + panic(err) + } +} + +func (p Profile) GetDatabaseVersion() (int, error) { + row := p.DB.QueryRow("select version_number from database_version") + + var version int + + err := row.Scan(&version) + if err != nil { + return 0, err + } + return version, nil +} + +func (p Profile) check_and_update_version() error { + version, err := p.GetDatabaseVersion() + if err != nil { + return err + } + + if version > ENGINE_DATABASE_VERSION { + return VersionMismatchError{ENGINE_DATABASE_VERSION, version} + } + + if ENGINE_DATABASE_VERSION > version { + fmt.Printf("Upgrading database from version %d to version %d", version, ENGINE_DATABASE_VERSION) + p.UpgradeFromXToY(version, ENGINE_DATABASE_VERSION) + } + + return nil +} + +/** + * Run all the migrations from version X to version Y, and update the `database_version` table's `version_number` + */ +func (p Profile) UpgradeFromXToY(x int, y int) error { + for i := x; i < y; i++ { + fmt.Println(MIGRATIONS[i]) + _, err := p.DB.Exec(MIGRATIONS[i]) + if err != nil { + return err + } + _, err = p.DB.Exec("update database_version set version_number = ?", i+1) + if err != nil { + return err + } + fmt.Printf("Now at database schema version %d.\n", i + 1) + } + return nil +} diff --git a/persistence/versions_test.go b/persistence/versions_test.go new file mode 100644 index 0000000..f229559 --- /dev/null +++ b/persistence/versions_test.go @@ -0,0 +1,33 @@ +package persistence_test + +import ( + "testing" + "os" + "offline_twitter/scraper" + "offline_twitter/persistence" +) + +func TestVersionUpgrade(t *testing.T) { + profile_path := "test_profiles/TestVersions" + if file_exists(profile_path) { + err := os.RemoveAll(profile_path) + if err != nil { + panic(err) + } + } + profile := create_or_load_profile(profile_path) + + test_migration := "insert into tweets (id, user_id, text) values (21250554358298342, -1, 'awefjk')" + test_tweet_id := scraper.TweetID(21250554358298342) + + if profile.IsTweetInDatabase(test_tweet_id) { + t.Fatalf("Test tweet shouldn't be in the database yet") + } + + persistence.MIGRATIONS = append(persistence.MIGRATIONS, test_migration) + profile.UpgradeFromXToY(persistence.ENGINE_DATABASE_VERSION, persistence.ENGINE_DATABASE_VERSION + 1) + + if !profile.IsTweetInDatabase(test_tweet_id) { + t.Errorf("Migration should have created the tweet, but it didn't") + } +}