From 01d47c6c826cc66343f00324b62d75459ec243d7 Mon Sep 17 00:00:00 2001 From: Jimmy Monin Date: Sun, 3 Jun 2018 16:09:00 +0200 Subject: [PATCH 1/6] Use PostgreSQL instead of MySQL --- scripts/_common.sh | 159 ++++++++++++++++++++++++++++++++++++++++++++- scripts/backup | 4 +- scripts/install | 11 ++-- scripts/remove | 4 +- scripts/restore | 10 +-- 5 files changed, 174 insertions(+), 14 deletions(-) diff --git a/scripts/_common.sh b/scripts/_common.sh index 7ce131e..dbe6ed5 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -3,7 +3,7 @@ # COMMON VARIABLES #================================================= -pkg_dependencies="php5-gd php5-json php5-intl php5-mcrypt php5-curl php5-apcu php5-redis php5-ldap php5-imagick imagemagick acl tar smbclient" +pkg_dependencies="php5-gd php5-json php5-intl php5-mcrypt php5-curl php5-apcu php5-redis php5-ldap php5-imagick imagemagick acl tar smbclient postgresql php-pgsql" if [ "$(lsb_release --codename --short)" != "jessie" ]; then pkg_dependencies="$pkg_dependencies php-zip php-apcu php-mbstring php-xml" @@ -334,7 +334,7 @@ ynh_multimedia_build_main_dir () { local checksum="4852c8607db820ad51f348da0dcf0c88" # Download yunohost.multimedia scripts - wget -nv https://github.com/YunoHost-Apps/yunohost.multimedia/archive/${ynh_media_release}.tar.gz + wget -nv https://github.com/YunoHost-Apps/yunohost.multimedia/archive/${ynh_media_release}.tar.gz # Verify checksum echo "${checksum} ${ynh_media_release}.tar.gz" | md5sum -c --status \ @@ -356,3 +356,158 @@ ynh_multimedia_addaccess () { groupadd -f multimedia usermod -a -G multimedia $user_name } + +#================================================= +# POSTGRESQL HELPERS +#================================================= + +# Create a master password and set up global settings +# Please always call this script in install and restore scripts +# +# usage: ynh_psql_test_if_first_run +ynh_psql_test_if_first_run() { + if [ -f /etc/yunohost/psql ]; + then + echo "PostgreSQL is already installed, no need to create master password" + else + pgsql=$(ynh_string_random) + pg_hba="" + echo "$pgsql" >> /etc/yunohost/psql + + if [ -e /etc/postgresql/9.4/ ] + then + pg_hba=/etc/postgresql/9.4/main/pg_hba.conf + elif [ -e /etc/postgresql/9.6/ ] + then + pg_hba=/etc/postgresql/9.6/main/pg_hba.conf + else + ynh_die "postgresql shoud be 9.4 or 9.6" + fi + + systemctl start postgresql + sudo --login --user=postgres psql -c"ALTER user postgres WITH PASSWORD '$pgsql'" postgres + + # force all user to connect to local database using passwords + # https://www.postgresql.org/docs/current/static/auth-pg-hba-conf.html#EXAMPLE-PG-HBA.CONF + # Note: we can't use peer since YunoHost create users with nologin + # See: https://github.com/YunoHost/yunohost/blob/unstable/data/helpers.d/user + sed -i '/local\s*all\s*all\s*peer/i \ + local all all password' "$pg_hba" + systemctl enable postgresql + systemctl reload postgresql + fi +} + +# Open a connection as a user +# +# example: ynh_psql_connect_as 'user' 'pass' <<< "UPDATE ...;" +# example: ynh_psql_connect_as 'user' 'pass' < /path/to/file.sql +# +# usage: ynh_psql_connect_as user pwd [db] +# | arg: user - the user name to connect as +# | arg: pwd - the user password +# | arg: db - the database to connect to +ynh_psql_connect_as() { + user="$1" + pwd="$2" + db="$3" + sudo --login --user=postgres PGUSER="$user" PGPASSWORD="$pwd" psql "$db" +} + +# # Execute a command as root user +# +# usage: ynh_psql_execute_as_root sql [db] +# | arg: sql - the SQL command to execute +# | arg: db - the database to connect to +ynh_psql_execute_as_root () { + sql="$1" + sudo --login --user=postgres psql <<< "$sql" +} + +# Execute a command from a file as root user +# +# usage: ynh_psql_execute_file_as_root file [db] +# | arg: file - the file containing SQL commands +# | arg: db - the database to connect to +ynh_psql_execute_file_as_root() { + file="$1" + db="$2" + sudo --login --user=postgres psql "$db" < "$file" +} + +# Create a database, an user and its password. Then store the password in the app's config +# +# After executing this helper, the password of the created database will be available in $db_pwd +# It will also be stored as "psqlpwd" into the app settings. +# +# usage: ynh_psql_setup_db user name [pwd] +# | arg: user - Owner of the database +# | arg: name - Name of the database +# | arg: pwd - Password of the database. If not given, a password will be generated +ynh_psql_setup_db () { + db_user="$1" + db_name="$2" + new_db_pwd=$(ynh_string_random) # Generate a random password + # If $3 is not given, use new_db_pwd instead for db_pwd. + db_pwd="${3:-$new_db_pwd}" + ynh_psql_create_db "$db_name" "$db_user" "$db_pwd" # Create the database + ynh_app_setting_set "$app" psqlpwd "$db_pwd" # Store the password in the app's config +} + +# Create a database and grant privilegies to a user +# +# usage: ynh_psql_create_db db [user [pwd]] +# | arg: db - the database name to create +# | arg: user - the user to grant privilegies +# | arg: pwd - the user password +ynh_psql_create_db() { + db="$1" + user="$2" + pwd="$3" + ynh_psql_create_user "$user" "$pwd" + sudo --login --user=postgres createdb --owner="$user" "$db" +} + +# Drop a database +# +# usage: ynh_psql_drop_db db +# | arg: db - the database name to drop +# | arg: user - the user to drop +ynh_psql_remove_db() { + db="$1" + user="$2" + sudo --login --user=postgres dropdb "$db" + ynh_psql_drop_user "$user" +} + +# Dump a database +# +# example: ynh_psql_dump_db 'roundcube' > ./dump.sql +# +# usage: ynh_psql_dump_db db +# | arg: db - the database name to dump +# | ret: the psqldump output +ynh_psql_dump_db() { + db="$1" + sudo --login --user=postgres pg_dump "$db" +} + + +# Create a user +# +# usage: ynh_psql_create_user user pwd [host] +# | arg: user - the user name to create +ynh_psql_create_user() { + user="$1" + pwd="$2" + sudo --login --user=postgres psql -c"CREATE USER $user WITH PASSWORD '$pwd'" postgres +} + +# Drop a user +# +# usage: ynh_psql_drop_user user +# | arg: user - the user name to drop +ynh_psql_drop_user() { + user="$1" + sudo --login --user=postgres dropuser "$user" +} diff --git a/scripts/backup b/scripts/backup index aecf0c0..884ac2f 100755 --- a/scripts/backup +++ b/scripts/backup @@ -55,10 +55,10 @@ ynh_backup "/etc/php5/fpm/pool.d/$app.conf" ynh_backup "/etc/php5/fpm/conf.d/20-$app.ini" #================================================= -# BACKUP THE MYSQL DATABASE +# BACKUP THE POSTGRESQL DATABASE #================================================= -ynh_mysql_dump_db "$db_name" > db.sql +ynh_psql_dump_db "$db_name" > /db.sql #================================================= # SPECIFIC BACKUP diff --git a/scripts/install b/scripts/install index 6900ed2..759c52e 100755 --- a/scripts/install +++ b/scripts/install @@ -60,12 +60,15 @@ ynh_app_setting_set $app user_home $user_home ynh_install_app_dependencies $pkg_dependencies #================================================= -# CREATE A MYSQL DATABASE +# CREATE A POSTGRESQL DATABASE #================================================= db_name=$(ynh_sanitize_dbid $app) +db_pwd=$(ynh_string_random) ynh_app_setting_set $app db_name $db_name -ynh_mysql_setup_db $db_name $db_name +ynh_app_setting_set $app db_pwd $db_pwd +ynh_psql_test_if_first_run +ynh_psql_create_db $db_name $db_name $db_pwd #================================================= # DOWNLOAD, CHECK AND UNPACK SOURCE @@ -134,7 +137,7 @@ chown -R $app: "$final_path" "$datadir" # Install Nextcloud using a temporary admin user exec_occ maintenance:install \ - --database "mysql" --database-name "$db_name" \ + --database "pgsql" --database-name "$db_name" \ --database-user "$db_name" --database-pass "$db_pwd" \ --admin-user "admin" --admin-pass "$(ynh_string_random 6)" \ --data-dir "$datadir" \ @@ -204,7 +207,7 @@ exec_occ config:system:get logout_url >/dev/null 2>&1 \ #================================================= # Set the user as admin -ynh_mysql_connect_as "$db_name" "$db_pwd" "$db_name" \ +ynh_psql_connect_as "$db_name" "$db_pwd" "$db_name" \ <<< "INSERT INTO oc_group_user VALUES ('admin','$admin');" # And delete admin user exec_occ user:delete admin diff --git a/scripts/remove b/scripts/remove index b21d030..fc6f82c 100755 --- a/scripts/remove +++ b/scripts/remove @@ -29,11 +29,11 @@ final_path=$(ynh_app_setting_get $app final_path) ynh_remove_app_dependencies #================================================= -# REMOVE THE MYSQL DATABASE +# REMOVE THE POSTGRESQL DATABASE #================================================= # Remove a database if it exists, along with the associated user -ynh_mysql_remove_db $db_name $db_name +ynh_psql_remove_db $db_name $db_name #================================================= # REMOVE APP MAIN DIR diff --git a/scripts/restore b/scripts/restore index 44a7f58..6bf31b4 100755 --- a/scripts/restore +++ b/scripts/restore @@ -33,6 +33,7 @@ domain=$(ynh_app_setting_get $app domain) path_url=$(ynh_app_setting_get $app path) final_path=$(ynh_app_setting_get $app final_path) db_name=$(ynh_app_setting_get $app db_name) +db_name=$(ynh_app_setting_get $app db_pwd) #================================================= # CHECK IF THE APP CAN BE RESTORED @@ -58,12 +59,13 @@ ynh_restore_file "/etc/nginx/conf.d/$domain.d/$app.conf" ynh_restore_file "$final_path" #================================================= -# RESTORE THE MYSQL DATABASE +# RESTORE THE POSTGRESQL DATABASE #================================================= -db_pwd=$(ynh_app_setting_get $app mysqlpwd) -ynh_mysql_setup_db $db_name $db_name $db_pwd -ynh_mysql_connect_as $db_name $db_pwd $db_name < ./db.sql +ynh_psql_test_if_first_run +ynh_psql_setup_db "$db_name" "$db_name" "$db_pwd" +# Restore dump +ynh_psql_execute_file_as_root ./db.sql "$db_name" #================================================= # RECREATE THE DEDICATED USER From 4d3635b8f5e9d6b7b6dfaa592f5d54c93f46f9b9 Mon Sep 17 00:00:00 2001 From: Jimmy Monin Date: Sun, 8 Jul 2018 09:18:01 +0200 Subject: [PATCH 2/6] Review fixes --- scripts/backup | 2 +- scripts/restore | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/backup b/scripts/backup index 884ac2f..634b63d 100755 --- a/scripts/backup +++ b/scripts/backup @@ -58,7 +58,7 @@ ynh_backup "/etc/php5/fpm/conf.d/20-$app.ini" # BACKUP THE POSTGRESQL DATABASE #================================================= -ynh_psql_dump_db "$db_name" > /db.sql +ynh_psql_dump_db "$db_name" > db.sql #================================================= # SPECIFIC BACKUP diff --git a/scripts/restore b/scripts/restore index 6bf31b4..920199d 100755 --- a/scripts/restore +++ b/scripts/restore @@ -33,7 +33,7 @@ domain=$(ynh_app_setting_get $app domain) path_url=$(ynh_app_setting_get $app path) final_path=$(ynh_app_setting_get $app final_path) db_name=$(ynh_app_setting_get $app db_name) -db_name=$(ynh_app_setting_get $app db_pwd) +db_pwd=$(ynh_app_setting_get $app db_pwd) #================================================= # CHECK IF THE APP CAN BE RESTORED From 86ad9bac8a7c042f5b861c70d31a79272a656d39 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 18 Dec 2020 11:22:02 +0100 Subject: [PATCH 3/6] Migrate from mysql to psql db --- manifest.json | 3 +-- scripts/_common.sh | 2 +- scripts/install | 2 +- scripts/upgrade | 18 ++++++++++++++++++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/manifest.json b/manifest.json index 5e3ec17..d333f6f 100644 --- a/manifest.json +++ b/manifest.json @@ -19,8 +19,7 @@ "multi_instance": true, "services": [ "nginx", - "php7.0-fpm", - "mysql" + "php7.3-fpm" ], "arguments": { "install" : [ diff --git a/scripts/_common.sh b/scripts/_common.sh index e6ba92f..2463abb 100644 --- a/scripts/_common.sh +++ b/scripts/_common.sh @@ -6,7 +6,7 @@ pkg_dependencies="imagemagick acl tar smbclient at postgresql" YNH_PHP_VERSION="7.3" -extra_php_dependencies="php${YNH_PHP_VERSION}-bz2 php${YNH_PHP_VERSION}-imap php${YNH_PHP_VERSION}-smbclient php${YNH_PHP_VERSION}-gmp php${YNH_PHP_VERSION}-gd php${YNH_PHP_VERSION}-json php${YNH_PHP_VERSION}-intl php${YNH_PHP_VERSION}-curl php${YNH_PHP_VERSION}-apcu php${YNH_PHP_VERSION}-redis php${YNH_PHP_VERSION}-ldap php${YNH_PHP_VERSION}-imagick php${YNH_PHP_VERSION}-zip php${YNH_PHP_VERSION}-mbstring php${YNH_PHP_VERSION}-xml php${YNH_PHP_VERSION}-mysql php${YNH_PHP_VERSION}-igbinary php${YNH_PHP_VERSION}-bcmath php${YNH_PHP_VERSION}-pgsql" +extra_php_dependencies="php${YNH_PHP_VERSION}-bz2 php${YNH_PHP_VERSION}-imap php${YNH_PHP_VERSION}-smbclient php${YNH_PHP_VERSION}-gmp php${YNH_PHP_VERSION}-gd php${YNH_PHP_VERSION}-json php${YNH_PHP_VERSION}-intl php${YNH_PHP_VERSION}-curl php${YNH_PHP_VERSION}-apcu php${YNH_PHP_VERSION}-redis php${YNH_PHP_VERSION}-ldap php${YNH_PHP_VERSION}-imagick php${YNH_PHP_VERSION}-zip php${YNH_PHP_VERSION}-mbstring php${YNH_PHP_VERSION}-xml php${YNH_PHP_VERSION}-igbinary php${YNH_PHP_VERSION}-bcmath php${YNH_PHP_VERSION}-pgsql" #================================================= # EXPERIMENTAL HELPERS diff --git a/scripts/install b/scripts/install index 3c120d4..ed052e4 100755 --- a/scripts/install +++ b/scripts/install @@ -65,7 +65,7 @@ ynh_script_progression --message="Creating a PostgreSQL database..." --weight=2 db_name=$(ynh_sanitize_dbid --db_name=$app) ynh_app_setting_set --app=$app --key=db_name --value=$db_name ynh_psql_test_if_first_run -ynh_psql_create_db --db_user=$db_name --db_name=$db_name +ynh_psql_setup_db --db_user=$db_name --db_name=$db_name #================================================= # DOWNLOAD, CHECK AND UNPACK SOURCE diff --git a/scripts/upgrade b/scripts/upgrade index 8c07855..f30fe14 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -230,6 +230,24 @@ datadir="/home/yunohost.app/$app/data" if [ "$upgrade_type" == "UPGRADE_APP" ] then + #================================================= + # HANDLE DATABASE MIGRATION + #================================================= + + mysql_db_pwd=$(ynh_app_setting_get --app=$app --key=mysqlpwd) + if [ -n "$mysql_db_pwd" ] + then + ynh_script_progression --message="Migrate Database..." + + ynh_psql_test_if_first_run + ynh_psql_setup_db --db_user=$db_name --db_name=$db_name + + exec_occ db:convert-type --all-apps pgsql $db_name 127.0.0.1 $db_name --password=$db_pwd -n + + ynh_mysql_remove_db --db_user=$db_name --db_name=$db_name + ynh_app_setting_delete --app=$app --key=mysqlpwd + fi + ynh_script_progression --message="Upgrading Nextcloud..." --weight=3 # Load the last available version From 9144036cdc3deaf10daaef927efaeabb170b7338 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 18 Dec 2020 14:21:52 +0100 Subject: [PATCH 4/6] fix restore --- scripts/restore | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/restore b/scripts/restore index 18c372d..b44aac7 100755 --- a/scripts/restore +++ b/scripts/restore @@ -44,6 +44,14 @@ test ! -d $final_path \ #================================================= # STANDARD RESTORATION STEPS +#================================================= +# REINSTALL DEPENDENCIES +#================================================= +ynh_script_progression --message="Reinstalling dependencies..." --weight=10 + +# Define and install dependencies +ynh_install_app_dependencies $pkg_dependencies + #================================================= # RESTORE THE NGINX CONFIGURATION #================================================= @@ -87,14 +95,6 @@ ynh_system_user_create --username=$app #================================================= # SPECIFIC RESTORATION -#================================================= -# REINSTALL DEPENDENCIES -#================================================= -ynh_script_progression --message="Reinstalling dependencies..." --weight=10 - -# Define and install dependencies -ynh_install_app_dependencies $pkg_dependencies - #================================================= # RESTORE THE PHP-FPM CONFIGURATION #================================================= From a67dbab962d8bf557f307816475ac0c8ac5bc9d4 Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 18 Dec 2020 14:22:17 +0100 Subject: [PATCH 5/6] db migration improvement --- scripts/upgrade | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/scripts/upgrade b/scripts/upgrade index f30fe14..8b1f234 100755 --- a/scripts/upgrade +++ b/scripts/upgrade @@ -92,6 +92,10 @@ then # Remove the post migration script before its execution ! ynh_exec_warn_less ynh_secure_remove --file="/tmp/owncloud_post_migration.sh" + # Remove both databases in case script fails during database migration + ynh_exec_warn_less ynh_psql_remove_db --db_user=$db_name --db_name=$db_name + ynh_exec_warn_less ynh_mysql_remove_db --db_user=$db_name --db_name=$db_name + # restore it if the upgrade fails ynh_restore_upgradebackup } @@ -228,26 +232,30 @@ local mount_id=$(exec_occ files_external:create --output=json \ # Define app's data directory datadir="/home/yunohost.app/$app/data" -if [ "$upgrade_type" == "UPGRADE_APP" ] +#================================================= +# HANDLE DATABASE MIGRATION +#================================================= + +mysql_db_pwd=$(ynh_app_setting_get --app=$app --key=mysqlpwd) +if [ -n "$mysql_db_pwd" ] then - #================================================= - # HANDLE DATABASE MIGRATION - #================================================= + ynh_script_progression --message="Migrate Database..." - mysql_db_pwd=$(ynh_app_setting_get --app=$app --key=mysqlpwd) - if [ -n "$mysql_db_pwd" ] - then - ynh_script_progression --message="Migrate Database..." + ynh_backup_if_checksum_is_different --file="$final_path/config/config.php" - ynh_psql_test_if_first_run - ynh_psql_setup_db --db_user=$db_name --db_name=$db_name + ynh_psql_test_if_first_run + ynh_psql_setup_db --db_user=$db_name --db_name=$db_name - exec_occ db:convert-type --all-apps pgsql $db_name 127.0.0.1 $db_name --password=$db_pwd -n + exec_occ db:convert-type --all-apps --clear-schema pgsql $db_name 127.0.0.1 $db_name --password=$db_pwd -n - ynh_mysql_remove_db --db_user=$db_name --db_name=$db_name - ynh_app_setting_delete --app=$app --key=mysqlpwd - fi + ynh_mysql_remove_db --db_user=$db_name --db_name=$db_name + ynh_app_setting_delete --app=$app --key=mysqlpwd + ynh_store_file_checksum --file="${final_path}/config/config.php" +fi + +if [ "$upgrade_type" == "UPGRADE_APP" ] +then ynh_script_progression --message="Upgrading Nextcloud..." --weight=3 # Load the last available version From 28fde19d799b357559753ccc9ec92afb028450ae Mon Sep 17 00:00:00 2001 From: Kay0u Date: Fri, 18 Dec 2020 17:33:24 +0100 Subject: [PATCH 6/6] add an upgrade test --- check_process | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/check_process b/check_process index 440eb43..7e8dd94 100644 --- a/check_process +++ b/check_process @@ -13,6 +13,7 @@ setup_public=0 upgrade=1 upgrade=1 from_commit=c864e086625363d91dde6ba78a652011a991a027 + upgrade=1 from_commit=e9e7015ed7667352c1c191fc632a9f6f382dd9d2 backup_restore=1 multi_instance=1 incorrect_path=1 @@ -25,3 +26,6 @@ Notification=none ; commit=c864e086625363d91dde6ba78a652011a991a027 name=[fix] php7.2-mcrypt doesn't exist (#253) manifest_arg=domain=DOMAIN&path=PATH&admin=USER&user_home=1& + ; commit=e9e7015ed7667352c1c191fc632a9f6f382dd9d2 + name=[fix] Upgrade to 20.0.4 (#346) + manifest_arg=domain=DOMAIN&path=PATH&admin=USER&user_home=1& \ No newline at end of file