設定三個節點的 MongoDB Cluster
在三台機器設定測試 MongoDB Cluster
環境設定
一個由三個 replica set 組成的 shard server，三個 config server，三個 route server。
shard1: 3 replica sets
192.168.1.11:27019
192.168.1.12:27019
192.168.1.13:27019
3 config server
192.168.1.11:27018
192.168.1.12:27018
192.168.1.13:27018
3 route server
192.168.1.11:27017
192.168.1.12:27017
192.168.1.13:27017
安裝
vi /etc/yum.repos.d/mongodb-org-4.2.repo
[mongodb-org-4.2]
name=MongoDB Repository
baseurl=https://repo.mongodb.org/yum/redhat/$releasever/mongodb-org/4.2/x86_64/
gpgcheck=1
enabled=1
gpgkey=https://www.mongodb.org/static/pgp/server-4.2.asc
yum install -y mongodb-org
# 安裝後，會修改的檔案
# mongodb system service
/usr/lib/systemd/system/mongod.service
# mongodb config file
/etc/mongod.conf
/usr/lib/systemd/system/mongod.service
[Unit]
Description=High-performance, schema-free document-oriented database
After=network.target
Documentation=https://docs.mongodb.org/manual
[Service]
User=mongod
Group=mongod
Environment="OPTIONS=-f /etc/mongod.conf"
ExecStart=/usr/bin/mongod $OPTIONS
ExecStartPre=/usr/bin/mkdir -p /var/run/mongodb
ExecStartPre=/usr/bin/chown mongod:mongod /var/run/mongodb
ExecStartPre=/usr/bin/chmod 0755 /var/run/mongodb
PermissionsStartOnly=true
PIDFile=/var/run/mongodb/mongod.pid
Type=forking
# file size
LimitFSIZE=infinity
# cpu time
LimitCPU=infinity
# virtual memory size
LimitAS=infinity
# open files
LimitNOFILE=64000
# processes/threads
LimitNPROC=64000
# locked memory
LimitMEMLOCK=infinity
# total threads (user+kernel)
TasksMax=infinity
TasksAccounting=false
# Recommended limits for for mongod as specified in
# http://docs.mongodb.org/manual/reference/ulimit/#recommended-settings
[Install]
WantedBy=multi-user.target
/etc/mongod.conf
# more /etc/mongod.conf
# mongod.conf
# for documentation of all options, see:
# http://docs.mongodb.org/manual/reference/configuration-options/
# where to write logging data.
systemLog:
destination: file
logAppend: true
path: /var/log/mongodb/mongod.log
# Where and how to store data.
storage:
dbPath: /var/lib/mongo
journal:
enabled: true
# engine:
# mmapv1:
# wiredTiger:
# how the process runs
processManagement:
fork: true # fork and run in background
pidFilePath: /var/run/mongodb/mongod.pid # location of pidfile
timeZoneInfo: /usr/share/zoneinfo
# network interfaces
net:
port: 27017
bindIp: 127.0.0.1 # Listen to local interface only, comment to listen on all interfaces.
#security:
#operationProfiling:
#replication:
#sharding:
## Enterprise-Only Options
#auditLog:
#snmp:
先將 server 停掉
sudo systemctl stop mongod
備份舊資料
mv /var/lib/mongo-cfgsvr /var/lib/mongo-cfgsvr.bak
mv /var/lib/mongo /var/lib/mongo.bak
mkdir -p /var/lib/mongo-cfgsvr
chown -R mongod:mongod /var/lib/mongo-cfgsvr
mkdir -p /var/lib/mongo
chown -R mongod:mongod /var/lib/mongo
修改 os 設定
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag
ssh 免密碼登入
/etc/hosts
192.168.1.11 larzio1
192.168.1.12 larzio2
192.168.1.13 larzio3
# ssh 免密碼
yum -y install openssh-clients
larzio1:
mkdir -p /root/.ssh
cd /root/.ssh
ssh-keygen -t dsa
按enter直到完成
(id_larzio1.pub為自己辨識用的名稱)
scp id_dsa.pub 192.168.1.12:/root/.ssh/id_larzio1.pub
scp id_dsa.pub 192.168.1.13:/root/.ssh/id_larzio1.pub
ssh 192.168.1.12(13)
cd /root/.ssh/
cat id_larzio1.pub >> authorized_keys
完成ssh免密碼
larzio2:
cd /root/.ssh
ssh-keygen -t dsa
按enter直到完成
(id_larzio2.pub為自己辨識用的名稱)
scp id_dsa.pub 192.168.1.11:/root/.ssh/id_larzio2.pub
scp id_dsa.pub 192.168.1.13:/root/.ssh/id_larzio2.pub
ssh 192.168.1.11(13)
cd /root/.ssh/
cat id_larzio2.pub >> authorized_keys
完成ssh免密碼
larzio3:
cd /root/.ssh
ssh-keygen -t dsa
按enter直到完成
(id_larzio3.pub為自己辨識用的名稱)
scp id_dsa.pub 192.168.1.11:/root/.ssh/id_larzio3.pub
scp id_dsa.pub 192.168.1.12:/root/.ssh/id_larzio3.pub
ssh 192.168.1.11(12)
cd /root/.ssh/
cat id_larzio3.pub >> authorized_keys
完成ssh免密碼
Note 備份, 還原 資料庫
mongodump -u root -p passwd --authenticationDatabase admin -d larzio -o /root/download/backup/
mongorestore -u root -p passwd --authenticationDatabase admin -d larzio --drop /root/download/backup/larzio
Config Server
步驟
- security key file
- config server 設定檔
- 產生 config server db path
- 啟動 config node service
- 部署到其他兩台機器
- 建立 replica set
security keyfile
openssl rand -base64 756 > /root/mongodb-keyfile
mkdir -p /var/lib/mongo
chown mongod.mongod /var/lib/mongo
cp -p /root/mongodb-keyfile /var/lib/mongo/
chmod 400 /var/lib/mongo/mongodb-keyfile
chown mongod.mongod /var/lib/mongo/mongodb-keyfile
# 複製到其他兩台機器
scp /var/lib/mongo/mongodb-keyfile root@192.168.1.12:/var/lib/mongo/mongodb-keyfile
scp /var/lib/mongo/mongodb-keyfile root@192.168.1.13:/var/lib/mongo/mongodb-keyfile
# 在 12, 13
sudo chmod 400 /var/lib/mongo/mongodb-keyfile
sudo chown mongod.mongod /var/lib/mongo/mongodb-keyfile
config file
cp -p /etc/mongod.conf /etc/mongod-cfgsvr.conf
vi /etc/mongod-cfgsvr.conf
# 修改以下設定
# systemLog.path
# storage.dbPath
# net.port
# net.bindIp
# security.keyFile
# sharding.clusterRole
# mongod.conf
# for documentation of all options, see:
# http://docs.mongodb.org/manual/reference/configuration-options/
# where to write logging data.
systemLog:
destination: file
logAppend: true
logRotate: reopen
path: /var/log/mongodb/mongod-cfgsvr.log
# Where and how to store data.
storage:
dbPath: /var/lib/mongo-cfgsvr
journal:
enabled: true
# engine:
# mmapv1:
# wiredTiger:
# how the process runs
processManagement:
fork: true # fork and run in background
pidFilePath: /var/run/mongodb/mongod-cfgsvr.pid # location of pidfile
timeZoneInfo: /usr/share/zoneinfo
# network interfaces
net:
port: 27018
bindIp: 0.0.0.0 # Listen to local interface only, comment to listen on all interfaces.
security:
keyFile: /var/lib/mongo/mongodb-keyfile
# authorization: enabled
#operationProfiling:
replication:
replSetName: rs-config
sharding:
clusterRole: configsvr
## Enterprise-Only Options
#auditLog:
#snmp:
config server dbpath
mkdir -p /var/lib/mongo-cfgsvr
chown -R mongod:mongod /var/lib/mongo-cfgsvr
config server service
# 不要用這個方式直接啟動, 所有產生的檔案owner 都會是 root:root
#mongod -f /etc/mongod-cfgsvr.conf
建立 mongod-cfgsvr service file
vi /usr/lib/systemd/system/mongod-cfgsvr.service
[Unit]
Description=High-performance, schema-free document-oriented database
After=network.target
Documentation=https://docs.mongodb.org/manual
[Service]
User=mongod
Group=mongod
Environment="OPTIONS=-f /etc/mongod-cfgsvr.conf"
ExecStart=/usr/bin/mongod $OPTIONS
ExecStartPre=/usr/bin/mkdir -p /var/run/mongodb
ExecStartPre=/usr/bin/chown mongod:mongod /var/run/mongodb
ExecStartPre=/usr/bin/chmod 0755 /var/run/mongodb
PermissionsStartOnly=true
PIDFile=/var/run/mongodb/mongod-cfgsvr.pid
Type=forking
# file size
LimitFSIZE=infinity
# cpu time
LimitCPU=infinity
# virtual memory size
LimitAS=infinity
# open files
LimitNOFILE=64000
# processes/threads
LimitNPROC=64000
# locked memory
LimitMEMLOCK=infinity
# total threads (user+kernel)
TasksMax=infinity
TasksAccounting=false
# Recommended limits for for mongod as specified in
# http://docs.mongodb.org/manual/reference/ulimit/#recommended-settings
[Install]
WantedBy=multi-user.target
啟動
systemctl daemon-reload
systemctl enable mongod-cfgsvr
systemctl start mongod-cfgsvr
deploy
複製到其他兩台機器
scp /etc/mongod-cfgsvr.conf root@192.168.1.12:/etc/mongod-cfgsvr.conf
scp /etc/mongod-cfgsvr.conf root@192.168.1.13:/etc/mongod-cfgsvr.conf
scp /usr/lib/systemd/system/mongod-cfgsvr.service root@192.168.1.13:/usr/lib/systemd/system/mongod-cfgsvr.service
scp /usr/lib/systemd/system/mongod-cfgsvr.service root@192.168.1.12:/usr/lib/systemd/system/mongod-cfgsvr.service
在 12, 13
mkdir -p /var/lib/mongo-cfgsvr
chown -R mongod:mongod /var/lib/mongo-cfgsvr
# 啟動 config-svr
systemctl daemon-reload
systemctl enable mongod-cfgsvr
systemctl start mongod-cfgsvr
replica set
先回到 192.168.1.11 關掉 config server，註解 replica, shard 的設定部分，以免 create user 發生 "no master" 的問題
# sudo mongod -f /etc/mongod-cfgsvr.conf -shutdown
systemctl stop mongod-cfgsvr
vi /etc/mongod-cfgsvr.conf
#註解掉 replica 與 shard
重新啟動 config server
#sudo mongod -f /etc/mongod-cfgsvr.conf
systemctl start mongod-cfgsvr
mongo -port 27018
use admin
db.createUser( {
user: "root",
pwd: "passwd",
roles: [ { role: "root", db: "admin" } ]
});
db.auth('root', 'passwd');
db.createUser( {
user: "admin",
pwd: "passwd",
roles: [ { role: "userAdminAnyDatabase", db: "admin" } ]
});
vi /etc/mongod-cfgsvr.conf
# 開啟 replica 與 shard
重新啟動服務
#sudo mongod -f /etc/mongod-cfgsvr.conf -shutdown
#sudo mongod -f /etc/mongod-cfgsvr.conf
systemctl restart mongod-cfgsvr
登入 mongo
mongo -port 27018 -u 'root' -p 'passwd' -authenticationDatabase 'admin'
rs.initiate(
{
_id: "rs-config",
configsvr: true,
members: [
{ _id : 0, host : "192.168.1.11:27018" },
{ _id : 1, host : "192.168.1.12:27018" },
{ _id : 2, host : "192.168.1.13:27018" }
]
}
)
rs.status()
Router Server
192.168.1.11, 192.168.1.12
config file
vi /etc/mongod-router.conf
# where to write logging data.
systemLog:
destination: file
logAppend: true
logRotate: reopen
path: /var/log/mongodb/mongod-router.log
processManagement:
fork: true # fork and run in background
pidFilePath: /var/run/mongodb/mongod-router.pid # location of pidfile
timeZoneInfo: /usr/share/zoneinfo
# network interfaces
net:
port: 27017
bindIp: 0.0.0.0 # Listen to local interface only, comment to listen on all interfaces.
security:
keyFile: /var/lib/mongo/mongodb-keyfile
sharding:
configDB: rs-config/192.168.1.11:27018,192.168.1.12:27018,192.168.1.13:27018
router service
建立 mongod-router service file
vi /usr/lib/systemd/system/mongod-router.service
[Unit]
Description=High-performance, schema-free document-oriented database
After=network.target
Documentation=https://docs.mongodb.org/manual
[Service]
User=mongod
Group=mongod
Environment="OPTIONS=-f /etc/mongod-router.conf"
ExecStart=/usr/bin/mongos $OPTIONS
ExecStartPre=/usr/bin/mkdir -p /var/run/mongodb
ExecStartPre=/usr/bin/chown mongod:mongod /var/run/mongodb
ExecStartPre=/usr/bin/chmod 0755 /var/run/mongodb
PermissionsStartOnly=true
PIDFile=/var/run/mongodb/mongod-router.pid
Type=forking
# file size
LimitFSIZE=infinity
# cpu time
LimitCPU=infinity
# virtual memory size
LimitAS=infinity
# open files
LimitNOFILE=64000
# processes/threads
LimitNPROC=64000
# locked memory
LimitMEMLOCK=infinity
# total threads (user+kernel)
TasksMax=infinity
TasksAccounting=false
# Recommended limits for for mongod as specified in
# http://docs.mongodb.org/manual/reference/ulimit/#recommended-settings
[Install]
WantedBy=multi-user.target
啟動 router
# sudo mongos -f /etc/mongod-router.conf
systemctl daemon-reload
systemctl enable mongod-router
systemctl start mongod-router
deploy
scp /etc/mongod-router.conf root@192.168.1.12:/etc/mongod-router.conf
scp /usr/lib/systemd/system/mongod-router.service root@192.168.1.12:/usr/lib/systemd/system/mongod-router.service
在 192.168.1.12
systemctl daemon-reload
systemctl enable mongod-router
systemctl start mongod-router
Shard Server
config file
vi /etc/mongod.conf
# mongod.conf
# 修改
# - net.port
# - net.bindIp
# - security.keyFile
# for documentation of all options, see:
# http://docs.mongodb.org/manual/reference/configuration-options/
# where to write logging data.
systemLog:
destination: file
logAppend: true
logRotate: reopen
path: /var/log/mongodb/mongod.log
# Where and how to store data.
storage:
dbPath: /var/lib/mongo
journal:
enabled: true
# engine:
# mmapv1:
# wiredTiger:
# how the process runs
processManagement:
fork: true # fork and run in background
pidFilePath: /var/run/mongodb/mongod.pid # location of pidfile
timeZoneInfo: /usr/share/zoneinfo
# network interfaces
net:
port: 27019
bindIp: 0.0.0.0
security:
keyFile: /var/lib/mongo/mongodb-keyfile
#operationProfiling:
#replication:
#sharding:
## Enterprise-Only Options
#auditLog:
在另外兩台 server 做一樣的設定
接下來，先啟動第一台 shard server，建立 replica set 之前必須要有 root user，否則會失敗。
scp /etc/mongod.conf root@192.168.1.12:/etc/mongod.conf
scp /etc/mongod.conf root@192.168.1.13:/etc/mongod.conf
啟動第一台 shard server
建立使用者
systemctl start mongod
mongo -port 27019
use admin
db.createUser( {
user: "root",
pwd: "passwd",
roles: [ { role: "root", db: "admin" } ]
});
db.auth('root', 'passwd');
db.createUser( {
user: "admin",
pwd: "passwd",
roles: [ { role: "userAdminAnyDatabase", db: "admin" } ]
});
關閉 mongod 服務
修改
vi /etc/mongod.conf
replication:
replSetName: rs-data
sharding:
clusterRole: shardsvr
啟動 mongod
在 192.168.1.12, 192.168.1.13 複製設定, 啟動 mogod
scp /etc/mongod.conf root@192.168.1.12:/etc/mongod.conf
scp /etc/mongod.conf root@192.168.1.13:/etc/mongod.conf
登入 mongo 並設定 replica
mongo -port 27019 -u 'root' -p 'passwd' -authenticationDatabase 'admin'
rs.initiate(
{
_id: "rs-data",
configsvr: false,
members: [
{ _id : 0, host : "192.168.1.11:27019" },
{ _id : 1, host : "192.168.1.12:27019" },
{ _id : 2, host : "192.168.1.13:27019" }
]
}
)
登入 router (mongos) 設定 shard
mongo -port 27017 -u 'root' -p 'passwd' -authenticationDatabase 'admin'
sh.addShard('rs-data/192.168.1.11:27019,192.168.1.12:27019,192.168.1.13:27019')
把認證模式給開啟，要加入認證模式的有 config 與 shard
開啟很簡單只要將 security.authorization 設定為 enabled
vi /etc/mongod-cfgsvr.conf
security:
keyFile: /var/lib/mongo/mongodb-keyfile
authorization: enabled
vi /etc/mongod.conf
security:
keyFile: /var/lib/mongo/mongodb-keyfile
authorization: enabled
scp /etc/mongod-cfgsvr.conf root@192.168.1.12:/etc/mongod-cfgsvr.conf
scp /etc/mongod-cfgsvr.conf root@192.168.1.13:/etc/mongod-cfgsvr.conf
scp /etc/mongod.conf root@192.168.1.12:/etc/mongod.conf
scp /etc/mongod.conf root@192.168.1.13:/etc/mongod.conf
重新啟動服務
systemctl restart mongod
systemctl restart mongod-cfgsvr
logrotate
vim /etc/logrotate.d/mongod
/var/log/mongodb/mongod.log {
daily
missingok
rotate 30
copytruncate
dateext
compress
notifempty
create 644 mongod mongod
sharedscripts
postrotate
/bin/kill -SIGUSR1 'cat /var/run/mongodb/mongod.pid 2> /dev/null' 2> /dev/null || true
endscript
}
vim /etc/logrotate.d/mongod-cfgsvr
/var/log/mongodb/mongod-cfgsvr.log {
daily
missingok
rotate 30
copytruncate
dateext
compress
notifempty
create 644 mongod mongod
sharedscripts
postrotate
/bin/kill -SIGUSR1 'cat /var/run/mongodb/mongod-cfgsvr.pid 2> /dev/null' 2> /dev/null || true
endscript
}
vim /etc/logrotate.d/mongod-router
/var/log/mongodb/mongod-router.log {
daily
missingok
rotate 30
copytruncate
dateext
compress
notifempty
create 644 mongod mongod
sharedscripts
postrotate
/bin/kill -SIGUSR1 'cat /var/run/mongodb/mongod-router.pid 2> /dev/null' 2> /dev/null || true
endscript
}
測試
logrotate -f -v /etc/logrotate.d/mongod
database user
在目標資料庫建立使用者
mongo -port 27017 -u 'root' -p 'passwd' -authenticationDatabase 'admin'
use larzio
db.createUser({
user: "larzio",
pwd: "passwd",
roles: [{ role: "readWrite", db: "larzio" }, { role: "dbAdmin", db: "larzio" }]
})
References
在 cent os 7上安裝 mongodb with Sharded Cluster (1) config server
在 cent os 7上安裝 mongodb with Sharded Cluster (2) router server 與 shard server
MongoDB Sharding 分散式儲存架構建置 (實作篇)
