2021/08/09

MongoDB Cluster

設定三個節點的 MongoDB Cluster

在三台機器設定測試 MongoDB Cluster

環境設定

一個由三個 replica set 組成的 shard server，三個 config server，三個 route server。

shard1: 3 replica sets
192.168.1.11:27019
192.168.1.12:27019
192.168.1.13:27019

3 config server
192.168.1.11:27018
192.168.1.12:27018
192.168.1.13:27018

3 route server
192.168.1.11:27017
192.168.1.12:27017
192.168.1.13:27017

安裝

vi /etc/yum.repos.d/mongodb-org-4.2.repo

[mongodb-org-4.2]
name=MongoDB Repository
baseurl=https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/4.2/x86_64/
gpgcheck=1
enabled=1
gpgkey=https://www.mongodb.org/static/pgp/server-4.2.asc

yum install -y mongodb-org

# 安裝後，會修改的檔案
# mongodb system service
/usr/lib/systemd/system/mongod.service
# mongodb config file
/etc/mongod.conf

/usr/lib/systemd/system/mongod.service

[Unit]
Description=High-performance, schema-free document-oriented database
After=network.target
Documentation=https://docs.mongodb.org/manual

[Service]
User=mongod
Group=mongod
Environment="OPTIONS=-f /etc/mongod.conf"
ExecStart=/usr/bin/mongod $OPTIONS
ExecStartPre=/usr/bin/mkdir -p /var/run/mongodb
ExecStartPre=/usr/bin/chown mongod:mongod /var/run/mongodb
ExecStartPre=/usr/bin/chmod 0755 /var/run/mongodb
PermissionsStartOnly=true
PIDFile=/var/run/mongodb/mongod.pid
Type=forking
# file size
LimitFSIZE=infinity
# cpu time
LimitCPU=infinity
# virtual memory size
LimitAS=infinity
# open files
LimitNOFILE=64000
# processes/threads
LimitNPROC=64000
# locked memory
LimitMEMLOCK=infinity
# total threads (user+kernel)
TasksMax=infinity
TasksAccounting=false
# Recommended limits for for mongod as specified in
# http://docs.mongodb.org/manual/reference/ulimit/#recommended-settings

[Install]
WantedBy=multi-user.target

/etc/mongod.conf

# more /etc/mongod.conf
# mongod.conf

# for documentation of all options, see:
#   http://docs.mongodb.org/manual/reference/configuration-options/

# where to write logging data.
systemLog:
  destination: file
  logAppend: true
  path: /var/log/mongodb/mongod.log

# Where and how to store data.
storage:
  dbPath: /var/lib/mongo
  journal:
    enabled: true
#  engine:
#  mmapv1:
#  wiredTiger:

# how the process runs
processManagement:
  fork: true  # fork and run in background
  pidFilePath: /var/run/mongodb/mongod.pid  # location of pidfile
  timeZoneInfo: /usr/share/zoneinfo

# network interfaces
net:
  port: 27017
  bindIp: 127.0.0.1  # Listen to local interface only, comment to listen on all interfaces.


#security:

#operationProfiling:

#replication:

#sharding:

## Enterprise-Only Options

#auditLog:

#snmp:

先將 server 停掉

sudo systemctl stop mongod

備份舊資料

mv /var/lib/mongo-cfgsvr /var/lib/mongo-cfgsvr.bak
mv /var/lib/mongo /var/lib/mongo.bak

mkdir -p /var/lib/mongo-cfgsvr
chown -R mongod:mongod /var/lib/mongo-cfgsvr

mkdir -p /var/lib/mongo
chown -R mongod:mongod /var/lib/mongo

修改 os 設定

echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag

ssh 免密碼登入

/etc/hosts

192.168.1.11   larzio1
192.168.1.12   larzio2
192.168.1.13   larzio3
# ssh 免密碼

yum -y install openssh-clients

larzio1:
mkdir -p /root/.ssh
cd /root/.ssh
ssh-keygen -t dsa
按enter直到完成

(id_larzio1.pub為自己辨識用的名稱)
scp id_dsa.pub 192.168.1.12:/root/.ssh/id_larzio1.pub
scp id_dsa.pub 192.168.1.13:/root/.ssh/id_larzio1.pub

ssh 192.168.1.12(13)
cd /root/.ssh/
cat id_larzio1.pub >> authorized_keys
完成ssh免密碼

larzio2:
cd /root/.ssh
ssh-keygen -t dsa
按enter直到完成

(id_larzio2.pub為自己辨識用的名稱)
scp id_dsa.pub 192.168.1.11:/root/.ssh/id_larzio2.pub
scp id_dsa.pub 192.168.1.13:/root/.ssh/id_larzio2.pub


ssh 192.168.1.11(13)
cd /root/.ssh/
cat id_larzio2.pub >> authorized_keys
完成ssh免密碼


larzio3:
cd /root/.ssh
ssh-keygen -t dsa
按enter直到完成

(id_larzio3.pub為自己辨識用的名稱)
scp id_dsa.pub 192.168.1.11:/root/.ssh/id_larzio3.pub
scp id_dsa.pub 192.168.1.12:/root/.ssh/id_larzio3.pub


ssh 192.168.1.11(12)
cd /root/.ssh/
cat id_larzio3.pub >> authorized_keys
完成ssh免密碼

Note 備份, 還原 資料庫

mongodump -u root -p passwd --authenticationDatabase admin -d larzio -o /root/download/backup/

mongorestore -u root -p passwd --authenticationDatabase admin -d larzio --drop /root/download/backup/larzio

Config Server

步驟

  1. security key file
  2. config server 設定檔
  3. 產生 config server db path
  4. 啟動 config node service
  5. 部署到其他兩台機器
  6. 建立 replica set

security keyfile

openssl rand -base64 756 > /root/mongodb-keyfile

mkdir -p /var/lib/mongo
chown mongod.mongod /var/lib/mongo

cp -p /root/mongodb-keyfile /var/lib/mongo/
chmod 400 /var/lib/mongo/mongodb-keyfile
chown mongod.mongod /var/lib/mongo/mongodb-keyfile

# 複製到其他兩台機器
scp /var/lib/mongo/mongodb-keyfile  root@192.168.1.12:/var/lib/mongo/mongodb-keyfile
scp /var/lib/mongo/mongodb-keyfile  root@192.168.1.13:/var/lib/mongo/mongodb-keyfile

# 在 12, 13
sudo chmod 400 /var/lib/mongo/mongodb-keyfile
sudo chown mongod.mongod /var/lib/mongo/mongodb-keyfile

config file

cp -p /etc/mongod.conf /etc/mongod-cfgsvr.conf

vi /etc/mongod-cfgsvr.conf
# 修改以下設定
#  systemLog.path
#  storage.dbPath
#  net.port
#  net.bindIp
#  security.keyFile
#  sharding.clusterRole

# mongod.conf

# for documentation of all options, see:
#   http://docs.mongodb.org/manual/reference/configuration-options/

# where to write logging data.
systemLog:
  destination: file
  logAppend: true
  logRotate: reopen
  path: /var/log/mongodb/mongod-cfgsvr.log

# Where and how to store data.
storage:
  dbPath: /var/lib/mongo-cfgsvr
  journal:
    enabled: true
#  engine:
#  mmapv1:
#  wiredTiger:

# how the process runs
processManagement:
  fork: true  # fork and run in background
  pidFilePath: /var/run/mongodb/mongod-cfgsvr.pid  # location of pidfile
  timeZoneInfo: /usr/share/zoneinfo

# network interfaces
net:
  port: 27018
  bindIp: 0.0.0.0  # Listen to local interface only, comment to listen on all interfaces.

security:
  keyFile: /var/lib/mongo/mongodb-keyfile
#  authorization: enabled

#operationProfiling:

replication:
  replSetName: rs-config

sharding:
  clusterRole: configsvr

## Enterprise-Only Options

#auditLog:

#snmp:

config server dbpath

mkdir -p /var/lib/mongo-cfgsvr
chown -R mongod:mongod /var/lib/mongo-cfgsvr

config server service

# 不要用這個方式直接啟動, 所有產生的檔案owner 都會是 root:root
#mongod -f /etc/mongod-cfgsvr.conf

建立 mongod-cfgsvr service file

vi /usr/lib/systemd/system/mongod-cfgsvr.service

[Unit]
Description=High-performance, schema-free document-oriented database
After=network.target
Documentation=https://docs.mongodb.org/manual

[Service]
User=mongod
Group=mongod
Environment="OPTIONS=-f /etc/mongod-cfgsvr.conf"
ExecStart=/usr/bin/mongod $OPTIONS
ExecStartPre=/usr/bin/mkdir -p /var/run/mongodb
ExecStartPre=/usr/bin/chown mongod:mongod /var/run/mongodb
ExecStartPre=/usr/bin/chmod 0755 /var/run/mongodb
PermissionsStartOnly=true
PIDFile=/var/run/mongodb/mongod-cfgsvr.pid
Type=forking
# file size
LimitFSIZE=infinity
# cpu time
LimitCPU=infinity
# virtual memory size
LimitAS=infinity
# open files
LimitNOFILE=64000
# processes/threads
LimitNPROC=64000
# locked memory
LimitMEMLOCK=infinity
# total threads (user+kernel)
TasksMax=infinity
TasksAccounting=false
# Recommended limits for for mongod as specified in
# http://docs.mongodb.org/manual/reference/ulimit/#recommended-settings

[Install]
WantedBy=multi-user.target

啟動

systemctl daemon-reload
systemctl enable mongod-cfgsvr
systemctl start mongod-cfgsvr

deploy

複製到其他兩台機器

scp /etc/mongod-cfgsvr.conf root@192.168.1.12:/etc/mongod-cfgsvr.conf
scp /etc/mongod-cfgsvr.conf root@192.168.1.13:/etc/mongod-cfgsvr.conf

scp /usr/lib/systemd/system/mongod-cfgsvr.service root@192.168.1.13:/usr/lib/systemd/system/mongod-cfgsvr.service

scp /usr/lib/systemd/system/mongod-cfgsvr.service root@192.168.1.12:/usr/lib/systemd/system/mongod-cfgsvr.service

在 12, 13

mkdir -p /var/lib/mongo-cfgsvr
chown -R mongod:mongod /var/lib/mongo-cfgsvr

# 啟動 config-svr
systemctl daemon-reload
systemctl enable mongod-cfgsvr
systemctl start mongod-cfgsvr

replica set

先回到 192.168.1.11 關掉 config server，註解 replica, shard 的設定部分，以免 create user 發生 "no master" 的問題

# sudo mongod -f /etc/mongod-cfgsvr.conf -shutdown
systemctl stop mongod-cfgsvr

vi /etc/mongod-cfgsvr.conf
#註解掉 replica 與 shard

重新啟動 config server

#sudo mongod -f /etc/mongod-cfgsvr.conf
systemctl start mongod-cfgsvr
mongo -port 27018

use admin

db.createUser( {
    user: "root",
    pwd: "passwd",
    roles: [ { role: "root", db: "admin" } ]
  });

db.auth('root', 'passwd');

db.createUser( {
    user: "admin",
    pwd: "passwd",
    roles: [ { role: "userAdminAnyDatabase", db: "admin" } ]
  });

vi /etc/mongod-cfgsvr.conf
# 開啟 replica 與 shard

重新啟動服務

#sudo mongod -f /etc/mongod-cfgsvr.conf -shutdown
#sudo mongod -f /etc/mongod-cfgsvr.conf
systemctl restart mongod-cfgsvr

登入 mongo

mongo -port 27018 -u 'root' -p 'passwd' -authenticationDatabase 'admin'

rs.initiate(
  {
    _id: "rs-config",
    configsvr: true,
    members: [
      { _id : 0, host : "192.168.1.11:27018" },
      { _id : 1, host : "192.168.1.12:27018" },
      { _id : 2, host : "192.168.1.13:27018" }
    ]
  }
)

rs.status()

Router Server

192.168.1.11, 192.168.1.12

config file

vi /etc/mongod-router.conf

# where to write logging data.
systemLog:
  destination: file
  logAppend: true
  logRotate: reopen
  path: /var/log/mongodb/mongod-router.log

processManagement:
  fork: true  # fork and run in background
  pidFilePath: /var/run/mongodb/mongod-router.pid  # location of pidfile
  timeZoneInfo: /usr/share/zoneinfo

# network interfaces
net:
  port: 27017
  bindIp: 0.0.0.0  # Listen to local interface only, comment to listen on all interfaces.


security:
  keyFile: /var/lib/mongo/mongodb-keyfile

sharding:
  configDB: rs-config/192.168.1.11:27018,192.168.1.12:27018,192.168.1.13:27018

router service

建立 mongod-router service file

vi /usr/lib/systemd/system/mongod-router.service

[Unit]
Description=High-performance, schema-free document-oriented database
After=network.target
Documentation=https://docs.mongodb.org/manual

[Service]
User=mongod
Group=mongod
Environment="OPTIONS=-f /etc/mongod-router.conf"
ExecStart=/usr/bin/mongos $OPTIONS
ExecStartPre=/usr/bin/mkdir -p /var/run/mongodb
ExecStartPre=/usr/bin/chown mongod:mongod /var/run/mongodb
ExecStartPre=/usr/bin/chmod 0755 /var/run/mongodb
PermissionsStartOnly=true
PIDFile=/var/run/mongodb/mongod-router.pid
Type=forking
# file size
LimitFSIZE=infinity
# cpu time
LimitCPU=infinity
# virtual memory size
LimitAS=infinity
# open files
LimitNOFILE=64000
# processes/threads
LimitNPROC=64000
# locked memory
LimitMEMLOCK=infinity
# total threads (user+kernel)
TasksMax=infinity
TasksAccounting=false
# Recommended limits for for mongod as specified in
# http://docs.mongodb.org/manual/reference/ulimit/#recommended-settings

[Install]
WantedBy=multi-user.target

啟動 router

# sudo mongos -f /etc/mongod-router.conf
systemctl daemon-reload
systemctl enable mongod-router
systemctl start mongod-router

deploy

scp /etc/mongod-router.conf root@192.168.1.12:/etc/mongod-router.conf

scp /usr/lib/systemd/system/mongod-router.service root@192.168.1.12:/usr/lib/systemd/system/mongod-router.service

在 192.168.1.12

systemctl daemon-reload
systemctl enable mongod-router
systemctl start mongod-router

Shard Server

config file

vi /etc/mongod.conf

# mongod.conf
# 修改
# - net.port
# - net.bindIp
# - security.keyFile

# for documentation of all options, see:
#   http://docs.mongodb.org/manual/reference/configuration-options/

# where to write logging data.
systemLog:
  destination: file
  logAppend: true
  logRotate: reopen
  path: /var/log/mongodb/mongod.log

# Where and how to store data.
storage:
  dbPath: /var/lib/mongo
  journal:
    enabled: true
#  engine:
#  mmapv1:
#  wiredTiger:

# how the process runs
processManagement:
  fork: true  # fork and run in background
  pidFilePath: /var/run/mongodb/mongod.pid  # location of pidfile
  timeZoneInfo: /usr/share/zoneinfo

# network interfaces
net:
  port: 27019
  bindIp: 0.0.0.0

security:
    keyFile: /var/lib/mongo/mongodb-keyfile

#operationProfiling:

#replication:

#sharding:

## Enterprise-Only Options

#auditLog:

在另外兩台 server 做一樣的設定 接下來，先啟動第一台 shard server，建立 replica set 之前必須要有 root user，否則會失敗。

scp /etc/mongod.conf root@192.168.1.12:/etc/mongod.conf
scp /etc/mongod.conf root@192.168.1.13:/etc/mongod.conf

啟動第一台 shard server

建立使用者

systemctl start mongod
mongo -port 27019

use admin

db.createUser( {
    user: "root",
    pwd: "passwd",
    roles: [ { role: "root", db: "admin" } ]
  });

db.auth('root', 'passwd');

db.createUser( {
    user: "admin",
    pwd: "passwd",
    roles: [ { role: "userAdminAnyDatabase", db: "admin" } ]
  });

關閉 mongod 服務

systemctl stop mongod

修改

vi /etc/mongod.conf

replication:
  replSetName: rs-data
    
sharding:
  clusterRole: shardsvr

啟動 mongod

systemctl start mongod

在 192.168.1.12, 192.168.1.13 複製設定, 啟動 mogod

scp /etc/mongod.conf root@192.168.1.12:/etc/mongod.conf
scp /etc/mongod.conf root@192.168.1.13:/etc/mongod.conf

systemctl start mongod

登入 mongo 並設定 replica

mongo -port 27019 -u 'root' -p 'passwd' -authenticationDatabase 'admin'

rs.initiate(
  {
    _id: "rs-data",
    configsvr: false,
    members: [
      { _id : 0, host : "192.168.1.11:27019" },
      { _id : 1, host : "192.168.1.12:27019" },
      { _id : 2, host : "192.168.1.13:27019" }
    ]
  }
)

登入 router (mongos) 設定 shard

mongo -port 27017 -u 'root' -p 'passwd' -authenticationDatabase 'admin'

sh.addShard('rs-data/192.168.1.11:27019,192.168.1.12:27019,192.168.1.13:27019')

把認證模式給開啟，要加入認證模式的有 config 與 shard 開啟很簡單只要將 security.authorization 設定為 enabled

vi /etc/mongod-cfgsvr.conf

security:
    keyFile: /var/lib/mongo/mongodb-keyfile
    authorization: enabled

vi /etc/mongod.conf

security:
    keyFile: /var/lib/mongo/mongodb-keyfile
    authorization: enabled
scp /etc/mongod-cfgsvr.conf root@192.168.1.12:/etc/mongod-cfgsvr.conf
scp /etc/mongod-cfgsvr.conf root@192.168.1.13:/etc/mongod-cfgsvr.conf

scp /etc/mongod.conf root@192.168.1.12:/etc/mongod.conf
scp /etc/mongod.conf root@192.168.1.13:/etc/mongod.conf

重新啟動服務

systemctl restart mongod
systemctl restart mongod-cfgsvr

logrotate

vim /etc/logrotate.d/mongod

/var/log/mongodb/mongod.log  {
    daily
    missingok
    rotate 30
    copytruncate
    dateext
    compress
    notifempty
    create 644 mongod mongod
    sharedscripts
    postrotate
        /bin/kill -SIGUSR1 'cat /var/run/mongodb/mongod.pid 2> /dev/null' 2> /dev/null || true
    endscript
}

vim /etc/logrotate.d/mongod-cfgsvr


/var/log/mongodb/mongod-cfgsvr.log {
    daily
    missingok
    rotate 30
    copytruncate
    dateext
    compress
    notifempty
    create 644 mongod mongod
    sharedscripts
    postrotate
        /bin/kill -SIGUSR1 'cat /var/run/mongodb/mongod-cfgsvr.pid 2> /dev/null' 2> /dev/null || true
    endscript
}

vim /etc/logrotate.d/mongod-router


/var/log/mongodb/mongod-router.log {
    daily
    missingok
    rotate 30
    copytruncate
    dateext
    compress
    notifempty
    create 644 mongod mongod
    sharedscripts
    postrotate
        /bin/kill -SIGUSR1 'cat /var/run/mongodb/mongod-router.pid 2> /dev/null' 2> /dev/null || true
    endscript
}

測試

logrotate -f -v /etc/logrotate.d/mongod

database user

在目標資料庫建立使用者

mongo -port 27017 -u 'root' -p 'passwd' -authenticationDatabase 'admin'

use larzio

db.createUser({
    user: "larzio",
    pwd: "passwd",
    roles: [{ role: "readWrite", db: "larzio" }, { role: "dbAdmin", db: "larzio" }]
})

References

在 cent os 7上安裝 mongodb with Sharded Cluster (1) config server

在 cent os 7上安裝 mongodb with Sharded Cluster (2) router server 與 shard server

MongoDB Sharding 分散式儲存架構建置 (實作篇)

於

沒有留言:

張貼留言