From 9519c9f54c0d132fb9bf40f84bac78adce8cd964 Mon Sep 17 00:00:00 2001 From: yukkop Date: Mon, 3 Feb 2025 20:06:13 +0000 Subject: [PATCH] feat(pg-from): dump schema from sqlite to postgres.sql --- package/postgres/pg-from/Cargo.lock | 189 ++++++++++++++++++ package/postgres/pg-from/Cargo.toml | 7 + package/postgres/pg-from/legacy_dump.sql | 168 ++++++++++++++++ package/postgres/pg-from/src/main.rs | 235 +++++++++++++++++++++++ 4 files changed, 599 insertions(+) create mode 100644 package/postgres/pg-from/Cargo.lock create mode 100644 package/postgres/pg-from/Cargo.toml create mode 100644 package/postgres/pg-from/legacy_dump.sql create mode 100644 package/postgres/pg-from/src/main.rs diff --git a/package/postgres/pg-from/Cargo.lock b/package/postgres/pg-from/Cargo.lock new file mode 100644 index 0000000..06e9cd3 --- /dev/null +++ b/package/postgres/pg-from/Cargo.lock @@ -0,0 +1,189 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "bitflags" +version = "2.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f68f53c83ab957f72c32642f3868eec03eb974d1fb82e453128456482613d36" + +[[package]] +name = "cc" +version = "1.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4730490333d58093109dc02c23174c3f4d490998c3fed3cc8e82d57afedb9cf" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] + +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e99fb7a497b1e3339bc746195567ed8d3e24945ecd636e3619d20b9de9e9149" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "pg-from" +version = "0.1.0" +dependencies = [ + "rusqlite", +] + +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + +[[package]] +name = "proc-macro2" +version = "1.0.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rusqlite" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "syn" +version = "2.0.98" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36147f1a48ae0ec2b5b3bc5b537d267457555a10dc06f3dbc8cb11ba3006d3b1" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a210d160f08b701c8721ba1c726c11662f877ea6b7094007e1ca9a1041945034" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/package/postgres/pg-from/Cargo.toml b/package/postgres/pg-from/Cargo.toml new file mode 100644 index 0000000..e3405a1 --- /dev/null +++ b/package/postgres/pg-from/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "pg-from" +version = "0.1.0" +edition = "2021" + +[dependencies] +rusqlite = { version = "0.32.0", features = ["bundled"] } diff --git a/package/postgres/pg-from/legacy_dump.sql b/package/postgres/pg-from/legacy_dump.sql new file mode 100644 index 0000000..d1d19e0 --- /dev/null +++ b/package/postgres/pg-from/legacy_dump.sql @@ -0,0 +1,168 @@ +-- PostgreSQL database dump generated from SQLite +CREATE SCHEMA IF NOT EXISTS legacy; + +SET client_encoding = 'UTF8'; + +CREATE TABLE legacy."promocode" ( + "promo_name" text NOT NULL, + "traffic_amount" bigint NOT NULL, + "remaining_activation" bigint NOT NULL, + "term" text, + "pool" text DEFAULT 'residential' +); + +ALTER TABLE legacy."promocode" OWNER TO postgres; + +CREATE TABLE legacy."user" ( + "user_id" text NOT NULL, + "buy_num" bigint DEFAULT 0, + "sub_id" bigint, + "used_promo_list" text, + "ip_list" text, + "pool" text +); + +ALTER TABLE legacy."user" OWNER TO postgres; + +CREATE TABLE legacy."price" ( + "gb_cost" bigint, + "pool" text NOT NULL DEFAULT 'residential', + "gb_cost_usd" double precision NOT NULL DEFAULT 0 +); + +ALTER TABLE legacy."price" OWNER TO postgres; + +CREATE TABLE legacy."all_user" ( + "user_id" text, + "lang" text, + "invited_by" text, + "ref_balance" bigint NOT NULL DEFAULT 0, + "pers_percent" text, + "reg_date" text, + "username" text, + "email" text, + "password" text, + "role_id" bigint, + "confirmed" bigint, + "tgcode" text, + "tgcode_expires" text, + "ref_balance_usd" double precision NOT NULL DEFAULT 0 +); + +ALTER TABLE legacy."all_user" OWNER TO postgres; + +CREATE UNIQUE INDEX idx_email ON legacy."all_user" ("email"); + +CREATE UNIQUE INDEX idx_username ON legacy."all_user" ("username"); + +CREATE TABLE legacy."admin_ref" ( + "value" text, + "name" text, + "number" bigint DEFAULT 0, + "user" text +); + +ALTER TABLE legacy."admin_ref" OWNER TO postgres; + +CREATE TABLE legacy."disc_promocode" ( + "name" text NOT NULL, + "discount" double precision NOT NULL, + "activations" bigint NOT NULL, + "first_use" text NOT NULL, + "term" text, + "user_for" text, + "is_global" bigint DEFAULT 0 +); + +ALTER TABLE legacy."disc_promocode" OWNER TO postgres; + +CREATE TABLE legacy."request" ( + "com" text, + "amount" bigint, + "user_id" text, + "username" text, + "in_id" SERIAL PRIMARY KEY +); + +ALTER TABLE legacy."request" OWNER TO postgres; + +CREATE TABLE legacy."system" ( + "key" text, + "value" text +); + +ALTER TABLE legacy."system" OWNER TO postgres; + +CREATE TABLE legacy."banner" ( + "name" text, + "photo_id" text, + "link" text +); + +ALTER TABLE legacy."banner" OWNER TO postgres; + +CREATE TABLE legacy."subuser" ( + "sub_id" bigint, + "owner_sub_id" bigint, + "label" text +); + +ALTER TABLE legacy."subuser" OWNER TO postgres; + +CREATE TABLE legacy."reseller" ( + "user_id" text, + "token" text, + "sub_id" bigint +); + +ALTER TABLE legacy."reseller" OWNER TO postgres; + +CREATE TABLE legacy."available_pay" ( + "name" text, + "is_available" text +); + +ALTER TABLE legacy."available_pay" OWNER TO postgres; + +CREATE TABLE legacy."payment" ( + "user_id" text NOT NULL, + "subuser_id" bigint NOT NULL, + "paid" bigint, + "order_id" text, + "amount_gb" bigint NOT NULL, + "balance_before" text NOT NULL, + "discount" text, + "service" text NOT NULL, + "date" text NOT NULL +); + +ALTER TABLE legacy."payment" OWNER TO postgres; + +CREATE TABLE legacy."temp_payment" ( + "result" text, + "payment_id" text, + "merchant_id" text, + "order_id" text, + "amount" bigint +); + +ALTER TABLE legacy."temp_payment" OWNER TO postgres; + +CREATE TABLE legacy."role" ( + "id" SERIAL PRIMARY KEY, + "name" text NOT NULL +); + +ALTER TABLE legacy."role" OWNER TO postgres; + +CREATE TABLE legacy."promo_activations" ( + "user_id" text NOT NULL, + "promo_name" text NOT NULL, + "usage_count" bigint NOT NULL DEFAULT 0, + PRIMARY KEY ("user_id", "promo_name") +); + +ALTER TABLE legacy."promo_activations" OWNER TO postgres; + +CREATE SEQUENCE legacy_request_seq START WITH 4 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; +CREATE SEQUENCE legacy_role_seq START WITH 3 INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1; diff --git a/package/postgres/pg-from/src/main.rs b/package/postgres/pg-from/src/main.rs new file mode 100644 index 0000000..0aed042 --- /dev/null +++ b/package/postgres/pg-from/src/main.rs @@ -0,0 +1,235 @@ +use rusqlite::{Connection, Result}; +use std::env; +use std::error::Error; +use std::fs::File; +use std::io::Write; + +/// Вывод справки по использованию утилиты. +fn print_help(program: &str) { + println!( + "Usage: {} \n\n\ + Options:\n -h, --help Show this help message\n\n\ + Example:\n {} mydb.sqlite legacy_dump.sql legacy", + program, program + ); +} + +/// Структура для хранения информации о столбце (результат PRAGMA table_info). +#[derive(Debug)] +struct ColumnInfo { + cid: i32, + name: String, + data_type: String, + notnull: bool, + dflt_value: Option, + pk: i32, +} + +/// Преобразует строку типа из SQLite в тип PostgreSQL. +/// Здесь применяется простая логика: если тип содержит "INT" – выдаётся bigint, +/// если содержит "CHAR"/"TEXT"/"CLOB" – text, если "REAL" или "FLOA"/"DOUB" – double precision, и т.д. +fn convert_sqlite_type_to_postgres(sqlite_type: &str) -> String { + let upper = sqlite_type.to_uppercase(); + if upper.contains("INT") { + "bigint".to_string() + } else if upper.contains("CHAR") || upper.contains("CLOB") || upper.contains("TEXT") { + "text".to_string() + } else if upper.contains("BLOB") { + "bytea".to_string() + } else if upper.contains("REAL") || upper.contains("FLOA") || upper.contains("DOUB") { + "double precision".to_string() + } else { + // значение по умолчанию + "text".to_string() + } +} + +/// Генерирует DDL для создания таблицы в PostgreSQL на основе информации из PRAGMA table_info. +fn generate_create_table_sql(table: &str, conn: &Connection, schema: &str) -> Result> { + let mut stmt = conn.prepare(&format!("PRAGMA table_info(\"{}\")", table))?; + let columns: Vec = stmt + .query_map([], |row| { + Ok(ColumnInfo { + cid: row.get(0)?, + name: row.get(1)?, + data_type: row.get(2)?, + notnull: row.get::<_, i32>(3)? != 0, + dflt_value: row.get(4)?, + pk: row.get(5)?, + }) + })? + .collect::, _>>()?; + + // Собираем список столбцов, а также определяем список первичных ключей. + let mut column_defs = Vec::new(); + let pk_columns: Vec<&ColumnInfo> = columns.iter().filter(|col| col.pk > 0).collect(); + + // Если имеется ровно один первичный ключ и его тип начинается с "INTEGER", + // то для него генерируем тип SERIAL (PostgreSQL автоматически создаст sequence). + let single_autoinc = if pk_columns.len() == 1 { + let col = pk_columns[0]; + col.data_type.to_uppercase().starts_with("INTEGER") + } else { + false + }; + + for col in &columns { + let mut col_def = format!("\"{}\" ", col.name); + if single_autoinc && pk_columns[0].name == col.name { + col_def.push_str("SERIAL PRIMARY KEY"); + } else { + let pg_type = convert_sqlite_type_to_postgres(&col.data_type); + col_def.push_str(&pg_type); + if col.notnull { + col_def.push_str(" NOT NULL"); + } + if let Some(default) = &col.dflt_value { + // Простейшая обработка значения по умолчанию; при необходимости можно доработать. + col_def.push_str(" DEFAULT "); + col_def.push_str(default); + } + // Если имеется ровно один pk и этот столбец является им, можно добавить PRIMARY KEY inline. + if pk_columns.len() == 1 && pk_columns[0].name == col.name { + col_def.push_str(" PRIMARY KEY"); + } + } + column_defs.push(col_def); + } + + // Если составной ключ (несколько столбцов с pk), добавляем ограничение отдельно. + if pk_columns.len() > 1 { + let pk_names: Vec = pk_columns + .iter() + .map(|col| format!("\"{}\"", col.name)) + .collect(); + let pk_def = format!("PRIMARY KEY ({})", pk_names.join(", ")); + column_defs.push(pk_def); + } + + let table_sql = format!( + "CREATE TABLE {}.\"{}\" (\n {}\n);", + schema, + table, + column_defs.join(",\n ") + ); + Ok(table_sql) +} + +/// Генерирует DDL для индексов таблицы. +/// Используется PRAGMA index_list и PRAGMA index_info для извлечения информации об индексах. +/// Автоиндексы (имена начинаются с "sqlite_autoindex") пропускаются. +fn generate_indexes_sql(table: &str, conn: &Connection, schema: &str) -> Result, Box> { + let mut indexes = Vec::new(); + let mut stmt = conn.prepare(&format!("PRAGMA index_list(\"{}\")", table))?; + let index_list = stmt.query_map([], |row| { + // row.get(1): имя индекса, row.get(2): флаг уникальности + let name: String = row.get(1)?; + let unique: i32 = row.get(2)?; + Ok((name, unique)) + })?; + for index_res in index_list { + let (index_name, unique) = index_res?; + // Пропускаем автоиндексы + if index_name.starts_with("sqlite_autoindex") { + continue; + } + // Получаем столбцы индекса + let mut stmt2 = conn.prepare(&format!("PRAGMA index_info(\"{}\")", index_name))?; + let cols_iter = stmt2.query_map([], |row| { + let col_name: String = row.get(2)?; + Ok(col_name) + })?; + let mut cols = Vec::new(); + for col_res in cols_iter { + cols.push(col_res?); + } + let unique_str = if unique != 0 { "UNIQUE " } else { "" }; + let index_sql = format!( + "CREATE {}INDEX {} ON {}.\"{}\" ({});", + unique_str, + index_name, + schema, + table, + cols.iter() + .map(|c| format!("\"{}\"", c)) + .collect::>() + .join(", ") + ); + indexes.push(index_sql); + } + Ok(indexes) +} + +fn main() -> Result<(), Box> { + // Обработка аргументов командной строки. + let args: Vec = env::args().collect(); + if args.iter().any(|arg| arg == "--help" || arg == "-h") { + print_help(&args[0]); + std::process::exit(0); + } + if args.len() < 4 { + eprintln!("Error: Insufficient arguments.\n"); + print_help(&args[0]); + std::process::exit(1); + } + let sqlite_file = &args[1]; + let output_file = &args[2]; + let schema = &args[3]; + + // Открываем SQLite БД. + let conn = Connection::open(sqlite_file)?; + + // Открываем (или создаём) выходной файл. + let mut out = File::create(output_file)?; + + // Записываем заголовок. + writeln!(out, "-- PostgreSQL database dump generated from SQLite")?; + writeln!(out, "CREATE SCHEMA IF NOT EXISTS {};\n", schema)?; + writeln!(out, "SET client_encoding = 'UTF8';\n")?; + + // Получаем имена таблиц (исключая внутренние). + let mut stmt = conn.prepare( + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'" + )?; + let table_names = stmt.query_map([], |row| row.get(0))?; + for table_name_result in table_names { + let table_name: String = table_name_result?; + // Генерируем DDL для таблицы + let table_sql = generate_create_table_sql(&table_name, &conn, schema)?; + writeln!(out, "{}\n", table_sql)?; + writeln!(out, "ALTER TABLE {}.\"{}\" OWNER TO postgres;\n", schema, table_name)?; + + // Генерируем DDL для индексов + let indexes = generate_indexes_sql(&table_name, &conn, schema)?; + for idx in indexes { + writeln!(out, "{}\n", idx)?; + } + } + + // Если имеется таблица sqlite_sequence, можно обработать автоинкрементные значения. + let sqlite_sequence_exists: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type='table' AND name='sqlite_sequence')", + [], + |row| row.get(0) + )?; + if sqlite_sequence_exists { + let mut stmt = conn.prepare("SELECT name, seq FROM sqlite_sequence")?; + let seq_iter = stmt.query_map([], |row| { + let table: String = row.get(0)?; + let seq: i64 = row.get(1)?; + Ok((table, seq)) + })?; + for seq_result in seq_iter { + let (table, seq) = seq_result?; + let seq_name = format!("{}_{}_seq", schema, table); + writeln!( + out, + "CREATE SEQUENCE {} START WITH {} INCREMENT BY 1 NO MINVALUE NO MAXVALUE CACHE 1;", + seq_name, + seq + 1 + )?; + } + } + + Ok(()) +}