598 tech add support for postgresql and pgsearch

Dockerfile

@@ -19,6 +19,7 @@ RUN echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources
 
 # init apt
 RUN apt-get update && apt-get upgrade -y
+
 # Install dependencies
 RUN apt-get install -y yarn rsync
 

Gemfile

@@ -7,6 +7,7 @@ gem 'rails', '~> 7.0'
 gem 'figaro'
 gem 'mysql2'
 gem 'net-ldap'
+gem 'pg'
 gem 'puma'
 gem 'sqlite3'
 
@@ -19,6 +20,7 @@ gem 'haml'
 gem 'maxmind-db'
 gem 'openid_connect'
 gem 'password_strength'
+gem 'pg_search'
 gem 'pundit'
 gem 'rails-i18n'
 gem 'seed-fu'

Gemfile.lock

@@ -200,6 +200,10 @@ GEM
       ast (~> 2.4.1)
     password_strength (1.1.4)
       activemodel
+    pg (1.3.5)
+    pg_search (2.3.6)
+      activerecord (>= 5.2)
+      activesupport (>= 5.2)
     pry (0.14.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -384,6 +388,8 @@ DEPENDENCIES
   net-ldap
   openid_connect
   password_strength
+  pg
+  pg_search
   pry
   pry-stack_explorer
   puma
@@ -406,4 +412,4 @@ DEPENDENCIES
   webmock
 
 BUNDLED WITH
-   2.2.32
+   2.2.33

app/models/encryptable.rb

@@ -27,6 +27,21 @@ class Encryptable < ApplicationRecord
   validates :name, presence: true
   validates :description, length: { maximum: 4000 }
 
+  scope :without_files, -> { where.not(type: Encryptable::File.sti_name) }
+
+  include PgSearch::Model
+  pg_search_scope :search_starts_with,
+                  against: [:name, :description],
+                  using: {
+                    tsearch: { prefix: true }
+                  }
+  include PgSearch::Model
+  pg_search_scope :search_starts_with,
+                  against: [:name, :description],
+                  using: {
+                    tsearch: { prefix: true }
+                  }
+
   def encrypt(_team_password)
     raise 'implement in subclass'
   end

app/models/folder.rb

@@ -28,6 +28,13 @@ class Folder < ApplicationRecord
   validates :name, length: { maximum: 70 }
   validates :description, length: { maximum: 300 }
 
+  include PgSearch::Model
+
+  pg_search_scope :search_starts_with,
+                  against: [:name, :description],
+                  using: {
+                    tsearch: { prefix: true }
+                  }
 
   def label
     name

app/models/team.rb

@@ -19,11 +19,23 @@ class Team < ApplicationRecord
   has_many :teammembers, dependent: :delete_all
   has_many :members, through: :teammembers, source: :user
   has_many :user_favourite_teams, dependent: :destroy
+  has_many :encryptables, through: :folders
 
   validates :name, presence: true
   validates :name, length: { maximum: 40 }
   validates :description, length: { maximum: 300 }
 
+  include PgSearch::Model
+  pg_search_scope :search_with_pg,
+                  against: [:name, :description],
+                  associated_against: {
+                    folders: [:name, :description],
+                    encryptables: [:name, :description]
+                  },
+                  using: {
+                    tsearch: { prefix: true }
+                  }
+
   def label
     name
   end
@@ -66,4 +78,5 @@ def create_teammember(user, plaintext_team_password)
     encrypted_team_password = Crypto::Rsa.encrypt(plaintext_team_password, user.public_key)
     teammembers.create!(password: encrypted_team_password, user: user)
   end
+
 end

app/presenters/filtered_list.rb

@@ -15,4 +15,8 @@ def fetch_entries
   def list_param(key)
     @params[key].to_a.map(&:to_i)
   end
+
+  def database_name
+    Rails.configuration.database_configuration[Rails.env]['database']
+  end
 end

app/presenters/teams/filtered_list.rb

@@ -57,7 +57,14 @@ def limit
     end
 
     def filter_by_query(teams)
-      teams.where(
+      if database_name.include? 'cryptopus-postgres'
+        @query_search = Teams::QuerySearch.new(@current_user, @params)
+      end
+      @query_search.filter_by_query(teams, query) || sql_filter_by_query(teams)
+    end
+
+    def sql_filter_by_query(teams)
+      teams.includes(:folders, folders: [:encryptables]).where(
         'lower(encryptables.description) LIKE :query
         OR lower(encryptables.name) LIKE :query
         OR lower(folders.name) LIKE :query

app/presenters/teams/query_search.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module ::Teams
+  class QuerySearch < ::FilteredList
+    def filter_by_query(teams, query)
+      teams.search_with_pg(query)
+    end
+  end
+end

bin/with_postgresql

@@ -0,0 +1,22 @@
+#!/bin/bash
+# prepare the environment to use the local postgresql database, e.g.
+#  bin /with_postgresql rails s
+#
+# rake tasks may use the following subtask:
+#  rake postgresql test:requests
+
+# if [ -z "$RAILS_ENV" ]
+# then
+#   env="development"
+# else
+#   env=$RAILS_ENV
+# fi
+
+# export RAILS_DB_ADAPTER=postgresql
+# export RAILS_DB_NAME=cryptopus_$env
+# export RAILS_DB_USERNAME=cryptopus
+# export RAILS_DB_PASSWORD=cryptopus
+# export RAILS_DB_HOST=127.0.0.1
+# export RAILS_DB_PORT=3306
+
+# $@

config/database.yml

@@ -6,19 +6,17 @@
 # PostgreSQL. Versions 8.2 and up are supported.
 #
 base: &generic
-  adapter: <%= ENV['RAILS_DB_ADAPTER'] || 'sqlite3' %>
+  adapter: <%= ENV['RAILS_DB_ADAPTER'] || 'postgresql' %>
   pool: 5
-  timeout: 5000
   encoding: utf8
 <% %w(host port username password).each do |option| %>
   <% value = ENV["RAILS_DB_#{option.upcase}"] %>
   <%= "#{option}: #{value}" if value.present? %>
 <% end %>
 
-
 development:
   <<: *generic
-  database: <%= ENV['RAILS_DB_NAME'] || "#{Rails.root}/db/development.sqlite3" %>
+  database: <%= "#{Rails.root}/db/cryptopus-postgres-development" %>
 
   # Connect on a TCP socket. Omitted by default since the client uses a
   # domain socket that doesn't need configuration. Windows does not have
@@ -40,7 +38,7 @@ development:
 # Do not set this db to the same as development or production.
 test:
   <<: *generic
-  database: <%= ENV['RAILS_DB_NAME'] || "#{Rails.root}/db/test.sqlite3" %>
+  database: <%= "#{Rails.root}/db/cryptopus-postgres-test" %>
 
 production:
   <<: *generic

config/database.yml-dist

@@ -12,7 +12,7 @@
 # And be sure to use new-style password hashing:
 #   http://dev.mysql.com/doc/refman/5.0/en/old-client.html
 development:
-  adapter: mysql
+  adapter: postgresql
   #socket: /var/lib/mysql/mysql.sock
   database: cryptopus_development
   username: cryptopus
@@ -23,15 +23,15 @@ development:
 # re-generated from your development database when you run 'rake'.
 # Do not set this db to the same as development or production.
 test:
-  adapter: mysql
+  adapter: postgresql
   #socket: /var/lib/mysql/mysql.sock
   database: cryptopus_test
   username: cryptopus
   password: yourpass
   host: localhost
 
 production:
-  adapter: mysql
+  adapter: postgresql
   #socket: /var/lib/mysql/mysql.sock
   database: cryptopus_production
   username: cryptopus

config/docker/keycloak-development/docker.env

@@ -21,7 +21,7 @@ POSTGRES_USER=keycloak
 POSTGRES_PASSWORD=password
 
 #Rails container
-RAILS_DB_ADAPTER=mysql2
+RAILS_DB_ADAPTER=postgresql
 RAILS_DB_HOST=cryptopus-mysql
 RAILS_DB_NAME=cryptopus_prod
 RAILS_DB_PORT=3306

config/docker/postgresql/docker-compose.yml

@@ -0,0 +1,30 @@
+version: '2'
+
+services:
+  cryptopus:
+    image: quay.io/puzzle/cryptopus:stable
+    container_name: cryptopus
+    env_file:
+      - postgresql-prod.env
+    environment:
+      RAILS_DB_ADAPTER: 'postgresql'
+      RAILS_DB_HOST: 'cryptopus-postgres'
+      RAILS_DB_NAME: 'cryptopus-postgres-db'
+      RAILS_DB_PORT: 3306
+      # RAILS_HOST_SSL: 'false' # uncomment for disabling ssl force
+    ports:
+      - "3306:3306"
+    links:
+      - cryptopus-postgresql:cryptopus-postgresql
+  cryptopus-postgres:
+    image: postgres:12.10
+    container_name: cryptopus-postgres
+    environment:
+      POSTGRES_PASSWORD: password
+    restart: always
+    ports:
+      - "5432:5432"
+    volumes:
+      - cryptopus-postgres:/var/lib/postgres
+volumes:
+  cryptopus-postgres:

config/docker/postgresql/postgresql-prod.env

@@ -0,0 +1,7 @@
+RAILS_DB_USERNAME=root
+RAILS_DB_PASSWORD=change-me
+SECRET_KEY_BASE=change-me
+#GEO_IP_LICENSE_KEY=change-me
+#SENTRY_DSN_FRONTEND=change-me
+# same value as RAILS_DB_PASSWORD
+MYSQL_ROOT_PASSWORD=change-me-too

db/migrate/20200612084506_change_file_column_size.rb

@@ -1,9 +1,9 @@
 class ChangeFileColumnSize < ActiveRecord::Migration[6.0]
   def up
-    change_column :file_entries, :file, :blob, size: :medium
+    change_column :file_entries, :file, :binary, size: :medium
   end
 
   def down
-    change_column :file_entries, :file, :blob
+    change_column :file_entries, :file, :binary
   end
 end

db/schema.rb

@@ -11,6 +11,9 @@
 # It's strongly recommended that you check this file into your version control system.
 
 ActiveRecord::Schema[7.0].define(version: 2022_03_29_122335) do
+  # These are extensions that must be enabled in order to support this database
+  enable_extension "plpgsql"
+
   create_table "encryptables", force: :cascade do |t|
     t.string "name", limit: 255, default: "", null: false
     t.integer "folder_id"
@@ -19,7 +22,7 @@
     t.datetime "updated_at", precision: nil, null: false
     t.string "tag"
     t.string "type", default: "Account::Credentials", null: false
-    t.text "encrypted_data", limit: 16777215
+    t.text "encrypted_data"
     t.integer "credential_id"
     t.text "content_type"
     t.index ["description"], name: "index_encryptables_on_description"
@@ -36,7 +39,7 @@
     t.index ["name"], name: "index_folders_on_name"
   end
 
-  create_table "settings", force: :cascade do |t|
+  create_table "settings", id: :serial, force: :cascade do |t|
     t.string "key", null: false
     t.string "value"
     t.string "type", null: false
@@ -69,7 +72,7 @@
     t.datetime "updated_at", null: false
   end
 
-  create_table "users", force: :cascade do |t|
+  create_table "users", id: :serial, force: :cascade do |t|
     t.text "public_key", null: false
     t.binary "private_key", null: false
     t.binary "password"
@@ -88,7 +91,7 @@
     t.integer "human_user_id"
     t.text "options"
     t.integer "role", default: 0, null: false
-    t.integer "default_ccli_user_id"
+    t.bigint "default_ccli_user_id"
     t.index ["default_ccli_user_id"], name: "index_users_on_default_ccli_user_id"
   end
 

docker-compose.yml

@@ -7,7 +7,12 @@ services:
     tty: true
     stdin_open: true
     environment:
-      - DISPLAY=${DISPLAY}
+      RAILS_DB_ADAPTER: 'postgresql'
+      RAILS_DB_HOST: 'cryptopus-postgres'
+      RAILS_DB_NAME: 'cryptopus-postgres-db'
+      RAILS_DB_USERNAME: 'postgres'
+      RAILS_DB_PASSWORD: 'password'
+      RAILS_DB_PORT: 5432
     build:
       context: ./config/docker/development
       dockerfile: Rails.dockerfile
@@ -38,6 +43,17 @@ services:
       - "4200:4200"
       - "7020:7020"
       - "7357:7357"
+  cryptopus-postgres:
+    image: postgres:12.10
+    container_name: cryptopus-postgres
+    environment:
+      POSTGRES_PASSWORD: password
+    restart: always
+    ports:
+      - "5432:5432"
+    volumes:
+      - cryptopus-postgres:/var/lib/postgres
 volumes:
   bundler_cache:
   yarn_cache:
+  cryptopus-postgres:

spec/controllers/api/admin/settings_controller_spec.rb

@@ -15,10 +15,11 @@
       get :index, xhr: true
 
       first_attributes = data.first['attributes']
-      expect(first_attributes['value']).to eq ['0.0.0.0', '192.168.10.0']
+      expect(first_attributes['value']).to eq ['CH', 'DE']
 
       second_attributes = data.second['attributes']
-      expect(second_attributes['value']).to eq %w[CH DE]
+      expect(second_attributes['value']).to eq ['0.0.0.0', '192.168.10.0']
+
       expect(data.size).to be(2)
       expect(included).to be(nil)
     end