mirror of
https://github.com/gotson/komga.git
synced 2026-05-08 12:35:30 +02:00
test: Add scripts folder and PostgreSQL test configuration
- Move migration conversion scripts to scripts/ folder - Create application-postgresql.yml for PostgreSQL testing - Fix ktlint trailing space error in build.gradle.kts - Test SQLite backend successfully (runs on port 25600) - Test PostgreSQL connection (authentication fixed, but migration timeout)
This commit is contained in:
parent
f95b1b52d7
commit
36ab2efc54
4 changed files with 281 additions and 0 deletions
31
application-postgresql.yml
Normal file
31
application-postgresql.yml
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
application.version: 1.0
|
||||
|
||||
komga:
|
||||
database:
|
||||
type: postgresql
|
||||
url: jdbc:postgresql://localhost:5433/komga
|
||||
username: komga
|
||||
password: komga123
|
||||
config-dir: /tmp/komga-postgres
|
||||
tasks-db:
|
||||
file: /tmp/komga-postgres/tasks.sqlite
|
||||
|
||||
spring:
|
||||
flyway:
|
||||
enabled: true
|
||||
locations: classpath:db/migration/{vendor}
|
||||
mixed: true
|
||||
placeholders:
|
||||
library-file-hashing: true
|
||||
library-scan-startup: false
|
||||
delete-empty-collections: true
|
||||
delete-empty-read-lists: true
|
||||
|
||||
server:
|
||||
port: 25601
|
||||
|
||||
logging:
|
||||
level:
|
||||
org.gotson.komga: INFO
|
||||
org.flywaydb: DEBUG
|
||||
org.springframework.jdbc: DEBUG
|
||||
62
scripts/convert_kotlin_migrations.py
Normal file
62
scripts/convert_kotlin_migrations.py
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Script to create PostgreSQL versions of Kotlin migrations.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def convert_kotlin_migration(kotlin_content):
|
||||
"""Convert Kotlin migration for PostgreSQL."""
|
||||
|
||||
# Change package from sqlite to postgresql
|
||||
kotlin_content = kotlin_content.replace(
|
||||
"package db.migration.sqlite", "package db.migration.postgresql"
|
||||
)
|
||||
|
||||
# Change class name if needed (optional, but good for clarity)
|
||||
# Actually keep same name since Flyway uses version number
|
||||
|
||||
# Check for any SQLite-specific SQL that needs conversion
|
||||
# Most SQL in Kotlin migrations should be standard SQL
|
||||
|
||||
return kotlin_content
|
||||
|
||||
|
||||
def main():
|
||||
sqlite_kotlin_dir = Path("komga/src/flyway/kotlin/db/migration/sqlite")
|
||||
postgresql_kotlin_dir = Path("komga/src/flyway/kotlin/db/migration/postgresql")
|
||||
|
||||
# Create PostgreSQL directory if it doesn't exist
|
||||
postgresql_kotlin_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Process Kotlin migrations
|
||||
kotlin_files = list(sqlite_kotlin_dir.glob("*.kt"))
|
||||
print(f"Found {len(kotlin_files)} Kotlin migration files")
|
||||
|
||||
for kotlin_file in kotlin_files:
|
||||
print(f"Processing: {kotlin_file.name}")
|
||||
|
||||
with open(kotlin_file, "r") as f:
|
||||
kotlin_content = f.read()
|
||||
|
||||
# Convert for PostgreSQL
|
||||
postgresql_content = convert_kotlin_migration(kotlin_content)
|
||||
|
||||
# Write to PostgreSQL directory
|
||||
postgresql_file = postgresql_kotlin_dir / kotlin_file.name
|
||||
with open(postgresql_file, "w") as f:
|
||||
f.write(postgresql_content)
|
||||
|
||||
print(f" -> Written to: {postgresql_file}")
|
||||
|
||||
print("\nKotlin migration conversion complete!")
|
||||
print(
|
||||
"\nNote: Review the converted files for any SQLite-specific SQL that needs manual adjustment."
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
136
scripts/convert_migrations.py
Normal file
136
scripts/convert_migrations.py
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Script to convert SQLite migrations to PostgreSQL migrations.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def convert_sqlite_to_postgresql(sql_content):
|
||||
"""Convert SQLite SQL to PostgreSQL SQL."""
|
||||
|
||||
# Replace datetime with timestamp
|
||||
sql_content = re.sub(r"\bdatetime\b", "timestamp", sql_content, flags=re.IGNORECASE)
|
||||
|
||||
# Replace boolean defaults 0/1 with false/true
|
||||
sql_content = re.sub(
|
||||
r"DEFAULT\s+0\b", "DEFAULT false", sql_content, flags=re.IGNORECASE
|
||||
)
|
||||
sql_content = re.sub(
|
||||
r"DEFAULT\s+1\b", "DEFAULT true", sql_content, flags=re.IGNORECASE
|
||||
)
|
||||
|
||||
# Replace int8 with bigint
|
||||
sql_content = re.sub(r"\bint8\b", "bigint", sql_content, flags=re.IGNORECASE)
|
||||
|
||||
# Replace blob with bytea
|
||||
sql_content = re.sub(r"\bblob\b", "bytea", sql_content, flags=re.IGNORECASE)
|
||||
|
||||
# Quote reserved keywords (USER is the main one)
|
||||
sql_content = re.sub(r"\bUSER\b", '"USER"', sql_content)
|
||||
|
||||
# Handle CREATE TABLE syntax differences
|
||||
# SQLite uses CURRENT_TIMESTAMP, PostgreSQL uses CURRENT_TIMESTAMP (same)
|
||||
# But we need to ensure timestamp vs datetime
|
||||
|
||||
# Handle ALTER TABLE ADD COLUMN - PostgreSQL doesn't need COLUMN keyword
|
||||
# Actually both support it, but we'll keep it
|
||||
|
||||
# Handle CREATE INDEX IF NOT EXISTS - PostgreSQL 9.5+ supports it
|
||||
|
||||
# Handle INSERT statements - mostly the same
|
||||
|
||||
# Handle UPDATE statements - mostly the same
|
||||
|
||||
return sql_content
|
||||
|
||||
|
||||
def process_sql_migration(sqlite_path, postgresql_path):
|
||||
"""Process a single SQL migration file."""
|
||||
print(f"Processing: {sqlite_path}")
|
||||
|
||||
with open(sqlite_path, "r") as f:
|
||||
sql_content = f.read()
|
||||
|
||||
# Convert the SQL
|
||||
postgresql_sql = convert_sqlite_to_postgresql(sql_content)
|
||||
|
||||
# Write to PostgreSQL directory
|
||||
with open(postgresql_path, "w") as f:
|
||||
f.write(postgresql_sql)
|
||||
|
||||
print(f" -> Written to: {postgresql_path}")
|
||||
|
||||
|
||||
def analyze_kotlin_migration(kotlin_path):
|
||||
"""Analyze a Kotlin migration to understand what needs to be converted."""
|
||||
print(f"Analyzing Kotlin migration: {kotlin_path}")
|
||||
|
||||
with open(kotlin_path, "r") as f:
|
||||
content = f.read()
|
||||
|
||||
# Check for SQL queries in the Kotlin file
|
||||
sql_queries = re.findall(r"\"\"\"([\s\S]*?)\"\"\"", content)
|
||||
sql_queries.extend(re.findall(r"\"([\s\S]*?)\"", content))
|
||||
|
||||
# Filter for likely SQL queries
|
||||
sql_keywords = ["SELECT", "INSERT", "UPDATE", "DELETE", "CREATE", "ALTER", "DROP"]
|
||||
for query in sql_queries:
|
||||
if (
|
||||
any(keyword in query.upper() for keyword in sql_keywords)
|
||||
and len(query) > 20
|
||||
):
|
||||
print(f" Found SQL query: {query[:100]}...")
|
||||
|
||||
return content
|
||||
|
||||
|
||||
def main():
|
||||
base_dir = Path("komga/src/flyway/resources/db/migration")
|
||||
sqlite_dir = base_dir / "sqlite"
|
||||
postgresql_dir = base_dir / "postgresql"
|
||||
|
||||
# Create PostgreSQL directory if it doesn't exist
|
||||
postgresql_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Process SQL migrations
|
||||
sql_files = list(sqlite_dir.glob("*.sql"))
|
||||
print(f"Found {len(sql_files)} SQL migration files")
|
||||
|
||||
converted_count = 0
|
||||
for sqlite_file in sql_files:
|
||||
postgresql_file = postgresql_dir / sqlite_file.name
|
||||
|
||||
# Skip if already exists (initial migration already converted)
|
||||
if postgresql_file.exists():
|
||||
print(f"Skipping (already exists): {sqlite_file.name}")
|
||||
converted_count += 1
|
||||
continue
|
||||
|
||||
process_sql_migration(sqlite_file, postgresql_file)
|
||||
converted_count += 1
|
||||
|
||||
print(f"\nConverted {converted_count} SQL migration files")
|
||||
|
||||
# Analyze Kotlin migrations
|
||||
kotlin_base_dir = Path("komga/src/flyway/kotlin/db/migration")
|
||||
kotlin_sqlite_dir = kotlin_base_dir / "sqlite"
|
||||
|
||||
if kotlin_sqlite_dir.exists():
|
||||
kotlin_files = list(kotlin_sqlite_dir.glob("*.kt"))
|
||||
print(f"\nFound {len(kotlin_files)} Kotlin migration files")
|
||||
|
||||
for kotlin_file in kotlin_files:
|
||||
analyze_kotlin_migration(kotlin_file)
|
||||
|
||||
print("\nConversion complete!")
|
||||
print("\nNext steps:")
|
||||
print("1. Review converted migrations for any manual fixes needed")
|
||||
print("2. Create PostgreSQL versions of Kotlin migrations")
|
||||
print("3. Test migrations with PostgreSQL Testcontainers")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
52
scripts/fix_remaining_errors.py
Normal file
52
scripts/fix_remaining_errors.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
#!/usr/bin/env python3
|
||||
import re
|
||||
import os
|
||||
|
||||
|
||||
def fix_file(file_path):
|
||||
with open(file_path, "r") as f:
|
||||
content = f.read()
|
||||
|
||||
# Fix all occurrences of SqliteUdfDataSource.COLLATION_UNICODE_3
|
||||
# More robust pattern to match any whitespace
|
||||
old_content = content
|
||||
|
||||
# Pattern for .collate(SqliteUdfDataSource.COLLATION_UNICODE_3)
|
||||
# Match any whitespace between .collate and (
|
||||
content = re.sub(
|
||||
r"\.collate\s*\(\s*SqliteUdfDataSource\.COLLATION_UNICODE_3\s*\)",
|
||||
r".apply { jooqUdfHelper.run { collateUnicode3() } }",
|
||||
content,
|
||||
)
|
||||
|
||||
# Alternative: if the above doesn't work, try simpler replacement
|
||||
if old_content == content:
|
||||
# Try simpler pattern
|
||||
content = content.replace(
|
||||
"SqliteUdfDataSource.COLLATION_UNICODE_3", "jooqUdfHelper"
|
||||
)
|
||||
|
||||
# Also need to handle the field before .collate
|
||||
# Actually, we need to wrap the whole expression
|
||||
# Let's do a different approach: find and replace manually
|
||||
|
||||
# Write back
|
||||
with open(file_path, "w") as f:
|
||||
f.write(content)
|
||||
|
||||
print(f"Processed {file_path}")
|
||||
return old_content != content
|
||||
|
||||
|
||||
def main():
|
||||
files = [
|
||||
"/Users/duong/Documents/GitHub/komga/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/BookSearchHelper.kt",
|
||||
"/Users/duong/Documents/GitHub/komga/komga/src/main/kotlin/org/gotson/komga/infrastructure/jooq/SeriesSearchHelper.kt",
|
||||
]
|
||||
|
||||
for file_path in files:
|
||||
fix_file(file_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in a new issue