Db on IT 运维小秋

mysql常用命令

chenwx716@139.com — Fri, 10 Feb 2023 10:00:00 +0800

备份

1. 逻辑备份


# 备份单库
mysqldump -u root -p -B db1 \
    --master-data=2 --single-transaction --flush-logs > db1-2020-05-09.sql

mysqldump -u wait -p \
    --single-transaction \
    --set-gtid-purged=OFF --databases db1 > local_wp.sqk


# 备份表结构
mysqldump -u root -p -d --add-drop-table db1 > 2020-05-11_db1.sql

mysqldump -u root -p --set-gtid-purged=OFF me_data_move > 0805-me_data_move.sql

# 只导出 insert 语句, 不生成创建表的动作
mysqldump --no-create-info db1 t1 t2 t3 > t_123.sql

# 备份 多个库
mysqldump -u root -p --set-gtid-purged=OFF -B db1 db2 db3 > 0806-all.sql

# 备份多个库
mysqldump -h db-xxx-uat.xxxx.ap-southeast-1.rds.amazonaws.com -P 13306 -u admin -p \
  --databases db1 db2 db3  > uat-db-20240813.sql

2. 逻辑恢复

# 恢复数据
mysql -u root -p < 0806-all.sql

# 恢复数据
mysql -u root -p $db_pw -S /data/mysql_13306/mysql.sock  < newsql.sql

对象管理

用户


-- 创建用户
CREATE USER 'user1'@'10.255.10.%' identified by 'passw0rd';

-- 早期方式
CREATE USER 'user1'@'10.255.10.%' IDENTIFIED WITH mysql_native_password;
grant all on db1.* to user1@'10.255.246.%' identified by '123456';

-- 删除用户
delete from user where user='user1' and host='10.255.246.%';

-- 刷新权限
flush privileges;

-- 查看权限
show grants for user@'%';

-- 授权
GRANT ALL ON *.* TO 'wait'@'%';

-- 修改密码
alter user 'root'@'localhost' identified by 'passw0rd';


set password=password('KdpTkiChO0MD!U&jyV@R');
-- 修改当前用户的密码

-- 单独设定指定用户的的连接数
ALTER USER 'root'@'%' WITH MAX_USER_CONNECTIONS 100;

实例


# 临时启动数据库
nohup mysqld_safe --defaults-file=/home/wait/mysqldb/my.cnf &

# 停止服务
mysqladmin -u root -p -P 3306 shutdown


# 关闭数据库 - 对于 mysqld_safe 启动的情况
mysqladmin -u root -p -S mysqldb/mysql.sock shutdown

-- 临时设置为只读 - 所有库
set read_only=on;
set super_read_only=on;

-- 临时配置
set global sync_binlog=0;
set global innodb_flush_log_at_trx_commit=0;
set global max_connections=8000;

PERSIST修改持久化

-- 持久化修改系统变量
-- 8.0+ 的 persist 将修改会持久化到临时文件中, 重启有效, 自动写入 mysqld-auto.cnf 文件
SET PERSIST max_connections = 200;

set persist read_only=on;
set persist super_read_only=on;

SET PERSIST_ONLY max_connections = 200;     -- 只持久化, 重启才生效

-- 清空持久化的配置
-- 只是清空 mysqld-auto.cnf 和 performance_schema.persisted_variables 中的内容
reset persist

库

show databases;               -- 显示所有数据库
use dbname;                   -- 进入数据库

create database if not exists localtest character set 'utf8mb4';


drop database db1;           -- 删除数据库

表

show tables;                  -- 显示all表

--- 修改表结构
alter table tablename modify id int(10) not null primary key auto_increment;

truncate table t_name;
drop table t_name;

desc tables;                    -- 查看表结构
show create table t_1;          -- 查看表 DDL


CREATE TABLE t1 (
    id INT PRIMARY KEY AUTO_INCREMENT,
    name VARCHAR(20),
    owner VARCHAR(20),
    sex CHAR(1),
    birth DATE
);

CREATE TABLE `t_test` (
    `id` INT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
    `time` DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT '入库时间',
    `key` VARCHAR(50) NOT NULL COMMENT '名称',
    `num` INTEGER NOT NULL default '0',
    `body` VARCHAR(50) NOT NULL COMMENT '名称'
);

-- 生成 TRUNCATE 全部表的 sql
SELECT CONCAT('TRUNCATE TABLE ', table_name, ';')
FROM information_schema.tables
WHERE table_schema = 'db1';

慢查询

1
2

-- 测试慢查询是否成功记录
select sleep(6);

开启审计日志

记录每一条sql, 开销比较大, 临时 debug

SET GLOBAL general_log = 1;

-- 查看审计日志文件路径
show variables like 'general%log%';

事务


-- 查看是否开启自动提交
show global variables like '%autocommit%';

-- 查看运行中的事务详细信息
select * from information_schema.INNODB_TRX \G;

-- 查看事务线程id和客户端信息
show processlist

-- 关闭某个线程id
kill 469273630

客户端

# 之后所有查询都会写入此文件
mysql> pager cat > /tmp/test.txt;

# login-path 设置快速登录
mysql_config_editor set --login-path=logdb --user=user1 --host=10.27.10.10 --port=3306 --password

mysql_config_editor print --all

mysql_config_editor remove --login-path=logdb

mysql --login-path=logdb    # 快捷登录

巡检

基础检查

select version();
show global status like 'uptime';   -- 查看运行时长
show databases;         -- 显示所有数据库
show processlist;       -- 查看会话信息
show warnings;          -- 查看最近的报错信息
show variables;         -- 查看所有参数变量
show global variables;
show global status;     -- 状态

show engine innodb status\G

网络

-- 连接数设置
show global variables like '%max_connect%';
show global variables like 'thread_cache_size%';     -- 线程池设定大小

show status like 'Connections';         -- 启动以来的连接总次数
show status like 'threads%';
SHOW GLOBAL STATUS LIKE 'Threads_connected';    -- 查看活跃连接数

-- 查看 root 用户的最大连接数量, - 如果为0则使用全局配置
SELECT USER, HOST, MAX_USER_CONNECTIONS FROM mysql.user WHERE USER = 'root';

-- 活跃用户 - 排除repl
select count(1) from processlist where state <> '' and user <> 'repl' and time > 2;

select * from socket_instances;
select * from performance_schema.users;
select * from performance_schema.hosts;

mysql> show status like 'threads%';
+-------------------+-------+
| Variable_name     | Value |
+-------------------+-------+
| Threads_cached    | 3     |       # 此刻已经建立的线程中有多少空闲线程
| Threads_connected | 807   |       # 当前已经建立的连接数
| Threads_created   | 810   |       # 代表从最近一次服务启动，已创建线程的数量
| Threads_running   | 2     |       # 代表当前激活的（非睡眠状态）线程数

IO-存储

mysql> show variables like 'log_bin';
mysql> show variables like 'sync_binlog';
mysql> show variables like 'innodb_flush_log_at_trx_commit';

SHOW GLOBAL VARIABLES LIKE 'open_files_limit';
SHOW GLOBAL VARIABLES LIKE 'flush';
SHOW GLOBAL VARIABLES LIKE 'sort_buffer_size';
SHOW GLOBAL VARIABLES LIKE '%buffer_size%';

show variables like 'innodb_commit_concurrency';
show variables like 'innodb_page_cleaners';
show variables like '%page_cleaner%';

SHOW GLOBAL VARIABLES LIKE 'innodb_buffer_pool_instances';
SHOW GLOBAL VARIABLES LIKE 'innodb_io_capacity';
SHOW GLOBAL VARIABLES LIKE 'innodb_lru_scan_depth';

库表查询

-- 查看数据库大小
SELECT
    table_schema AS '数据库名',
    ROUND(SUM(data_length + index_length) / 1024 / 1024, 2) AS '大小(MB)',
    ROUND(SUM(data_length) / 1024 / 1024, 2) AS '数据大小(MB)',
    ROUND(SUM(index_length) / 1024 / 1024, 2) AS '索引大小(MB)'
FROM information_schema.tables
GROUP BY table_schema
ORDER BY SUM(data_length + index_length) DESC;


-- 查看数据库大小 - 含表数量
SELECT
    table_schema AS 'Database',
    COUNT(*) AS 'Tables',
    ROUND(SUM(data_length) / 1024 / 1024, 2) AS 'Data Size (MB)',
    ROUND(SUM(index_length) / 1024 / 1024, 2) AS 'Index Size (MB)',
    ROUND(SUM(data_length + index_length) / 1024 / 1024, 2) AS 'Total Size (MB)',
    ROUND(SUM(data_free) / 1024 / 1024, 2) AS 'Free Space (MB)'
FROM information_schema.tables
WHERE table_schema NOT IN ('information_schema', 'mysql', 'performance_schema', 'sys')
GROUP BY table_schema
ORDER BY SUM(data_length + index_length) DESC;


-- 查询表行数 和 数据大小 和 索引大小
SELECT table_schema, table_name, table_rows,
    data_length/1024/1024 as SIZE_T_MB,
    index_length/1024/1024 as SIZE_I_MB
FROM information_schema.tables WHERE table_schema = 'mytest';


-- 查询自增id的使用情况
select
    table_schema,table_name,engine,Auto_increment
from
    information_schema.tables
where
    INFORMATION_SCHEMA.tables.TABLE_SCHEMA not in ("information_schema","performance_schema","sys","mysql");

故障处理

`1`	`mysql -uroot -p -e "show processlist" \|grep -v Sleep`

`1`	SELECT * FROM information_schema.`PROCESSLIST` WHERE command != "Sleep" ORDER BY time DESC;

结束阻塞任务

-- 生成 kill 执行时间超过 5 分钟的sql
SELECT
    concat( 'kill ', id, ';' )
FROM
    information_schema.PROCESSLIST
WHERE
    command != 'Sleep'
    AND time > 5 * 60
ORDER BY
    time DESC;

-- 查看当前未关闭的事务
select * from information_schema.INNODB_TRX \G;

sql语句

插入更新删除

INSERT INTO t1 VALUES ('cwx', 'chenwx', 'n', '1990-03-30');

insert into tablesname(字段1,字段2,字段3) values(值,值,值);
insert into tablesname(字段1,字段2,字段3) values(值,值,值),(值,值,值),(值,值,值);

UPDATE t1 SET name = 'cwx1' where owner = 'chenwx';

DELETE FROM t1 where name = 'cwx';

简单示例

select * from t_name limit 5;

-- 去重显示(即每项目只显示一次)
SELECT DISTINCT c1 FROM teacher;


-- 多查询
SELECT * FROM t1 WHERE degree IN (85, 86, 88);

-- 区间查询
SELECT * FROM t1 WHERE degree BETWEEN 60 AND 80;
SELECT * FROM t1 WHERE degree > 60 AND degree < 80;

-- 按月分组
SELECT DATE_FORMAT(insert_time,'%Y%m') months,count(id) from t1 group by months;

-- 分渠道去重
select  t.spid,count(distinct t.login_name) from t1 t group by t.spid;


-- 子查询
select name,math from t1 where math=(select max(math) from t1);
select a,b,c from a where a IN (select d from b where c='3');

select a.name from
    (select * from table_name where classess="数学") a,
    (select * from table_name where classess="语文") b
    where a.court<b.court and a.name=b.name;

-- 统计
SELECT
    SUM(CASE WHEN is_logoff = 0 THEN 1 ELSE 0 END) AS 有效用户,
    SUM(CASE WHEN is_logoff = 1 THEN 1 ELSE 0 END) AS 注销用户
FROM
    t_user;

较复杂


-- 查询重复行
SELECT ctime, COUNT(*) AS count FROM t1 GROUP BY ctime HAVING COUNT(*) > 1;

-- 列出重复行
SELECT * FROM t1 WHERE ctime IN (SELECT ctime FROM t1 GROUP BY ctime HAVING COUNT(*) > 1);


-- 查找具有重复 code 值的条目
select * from products group by code having count(code)>1;
    -- 按 group by 分组后, 就已经完成了合并
    -- having count(code)>1  表示过滤出数量大于 1 条的内容

-- 查找重复的数据
select * from t_host_info where 1=1 group by random_num having count(random_num) >1;

转换

-- 类型转化
CAST(t.body AS CHAR)

inet_aton()     -- 将ip地址转换成数字型 10.1.249.55 -> 167901495
inet_ntoa()     -- 将数字型转换成ip地址

主从

show master status;         -- 查看当前binlog位置
show slave status\G         -- 查看复制状态
show master logs;           -- 查看 binlog 文件
flush logs;                 -- 轮转binlog日志

-- 清理之前的 binlog
PURGE MASTER LOGS TO 'mysql-binlog.000336';

reset master;

-- 查看跳过错误的配置
show variables like 'slave_skip_errors';

show variables like '%gtid%';

-- 创建
CHANGE MASTER TO
  MASTER_HOST='10.2.1.5',
  MASTER_PORT=3306,
  MASTER_USER='sync',
  MASTER_PASSWORD='passw0rd',
  GET_MASTER_PUBLIC_KEY=1,              -- 8.x 非 ssl 同步时需要获取公钥
  MASTER_LOG_FILE='binlog.000022',
  MASTER_LOG_POS=476432;

`1`	`mysql -u root -p -e "show slave status\G;" \| grep _Running`

nosql-工具箱

chenwx716@139.com — Fri, 10 Feb 2023 10:00:00 +0800

etcd

# 创建快照备份
export ETCDCTL_API=3
etcdctl --endpoints localhost:2379 snapshot save snapshot.db \
          --cacert=/etc/kubernetes/pki/etcd/ca.crt  \
          --cert=/etc/kubernetes/pki/etcd/server.crt \
          --key=/etc/kubernetes/pki/etcd/server.key

# 快照恢复命令
etcdctl snapshot restore snapshot.db --name m3 --data-dir=/home/etcd_data

memached

# run
memcached -d -p 39001 -m 30 -P memcached.pid

# 清空缓存
echo flush_all | nc 10.2.1.5 21101

# show
echo stats | nc 10.2.1.5 21101
echo stats | nc 10.2.1.5 21101 | grep curr_connections

# 查看 lru 淘汰的key数量
echo stats | nc 10.2.1.5 21101 | grep evictions

# get
echo -ne "get key_name\r\n" | nc 10.2.1.5 21101

# del
echo "delete jtsysconfig_traffic_control_config_12028" | nc 10.2.1.5 21101

# set
printf "set key008 0 5 10\r\noldboy0987\r\n"|nc 10.2.1.5 11211

set wait 0 180 9
chenwx716

consul


consul acl set-agent-token default 0bc6bc46-f25e-4262-b2d9-ffbe1d96be6f

consul agent -dev

consul agent -config-dir config

consul agent -server -bootstrap -bind=127.0.0.1 -client=127.0.0.1 -node=redis1 \
    -config-dir conf -enable-script-checks -data-dir=data -ui -node=127.0.0.1


http://127.0.0.1:8500

consul.exe agent -server -bootstrap-expect 1 \
    -data-dir "D://app//consul_1.11.2//data" \
    -advertise 127.0.0.1 -ui -node=node1 \
    -bind 127.0.0.1 -client 0.0.0.0 -rejoin -config-dir "D://app//consul_1.11.2//config"

es


curl 'http://localhost:9200/_cat/indices?v'

# 获取全部数据，默认只显示10条
curl -XGET 'http://localhost:9200/index-name/doc-type/_search?pretty'

# 根据ID查询数据
curl -XGET 'http://localhost:9200/index-name/doc-type/id?pretty'

# 查询条件，参数 q=
curl -XGET 'http://localhost:9200/megacorp/employee/_search?q=name:Smith'

curl -XGET 'http://localhost:29200/nginx_access/_search?q=x_forwarded_for:117.136.38.182&pretty'

# del index
curl -X DELETE "http://localhost:19200/indexName?pretty"'&timeout=5m'

# 时间范围数量统计
curl -XGET 'http://10.27.10.30:29200/cd-nginx-2020.11.01/_count?pretty' \
-H 'content-type: application/json' -d '
{
  "query": {
    "range":{
      "@timestamp":{
        "gte": "2020-11-01 10:00:00",
        "lte": "2020-11-01 11:00:00",
        "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd",
        "time_zone": "+08:00"
      }
    }
  }
}'

kibana

常用查询语句

1
2

# 指定路径+耗时范围
request_uri:"/services/LoginForApp" AND request_time:[0.500 TO 9.000]

redis

慢查询


slowlog get                                     # 查询慢查询记录

# 配置
config set slowlog-log-slower-than 1000         # 1ms
config set slowlog-max-len 500
CONFIG GET slowlog-*                            # 获取配置

客户端

# 连接集群
redis-cli --cluster 10.27.10.1 21101
redis-cli --cluster -p 21101
redis-cli -c -h 10.27.10.1 -p 11102
redis-cli -c -h 192.168.0.1 -p 6389 -a password

数据管理

FLUSHALL                # 清空所有库
FLUSHDB                 # 清空单个库 - 当前库

DBSIZE                  # show key number
TTL key                 # 获取key的剩余过期时间s

监控

redis-cli MONITOR

MONITOR
INFO
STATS

CLUSTER info
CLIENT LIST
cluster nodes

集群管理

# 添加从节点
./redis-cli --cluster add-node <新节点> <连接的主节点> --cluster-slave --cluster-master-id <主节点ID>

./redis-cli --cluster add-node 10.1.99.91:16483 10.1.99.70:16379 --cluster-slave --cluster-master-id feb7b92d1a183b14e5528c667e58592d8131057b

sqlite

.databases              # show current db
.tables                 # show all tables
.indices [table]        # 获取表的索引列
.schema [table]         # 显示表结构 DDL

# 非交互查询
sqlite3 test.db "sleect * from t_file"

postgresql工具箱

chenwx716@139.com — Fri, 10 Feb 2023 10:00:00 +0800

备份-恢复

备份-pg_dump

export PGPASSWORD='user123456'

# pg_dump 逻辑备份工具, 支持二进制导出

# 关键参数
-O      不指定对象的所有者
-c      在重新创建前, 先删除对象
-x      不转储权限信息
-F d    目录存储
-b      二进制
-f      指定位置

# 普通逻辑备份
pg_dump -h 172.27.0.50 -U user1 -d db1 > db1.sql

# 备份表结构
pg_dump -U user1 -s db1 > db1_table_info.sql

# 不包括权限和属主信息, 包括删除对象的操作
pg_dump -U user1 -d db1 -x -O -c -f db1.sql

# 二进制, 目录格式 - 推荐
pg_dump -U user1 -d db1 -F d -b -c -f /backup/db1-20250904-1431
pg_dump -U user1 -d db1 -F d -b -x -O -c -f /backup/db1-20250904-1431

# 备份时并忽略指定表
pg_dump -U user1 -d db1 -O -x -c \
    --exclude-table-data=pre_commit \
    --exclude-table-data=tm_validators -f pro_db1.sql

恢复-逻辑恢复

`1`	`psql -U user1 -d db1 -f 08281925_data.sql`

管理

用户

`1`	`create user user1 with password 'passw0rd';`

库


-- 创建库并赋权限
CREATE DATABASE db1;
GRANT ALL PRIVILEGES ON DATABASE db1 TO user1;


-- 查看数据库大小
SELECT pg_size_pretty(pg_database_size('me_hub'));

SCHEMA

`1`	`DROP SCHEMA myschema CASCADE; -- 删除模式和其下的对象`

客户端


# 查看数据库列表
psql -Upostgres -c '\l'


# 主机上安装不了高版本 psql 时, 使用 docker  客户端
docker run --name rds_tmp_task1 --rm -it \
    -e "PGPASSWORD=user123456" \
    -v /backup:/backup \
    -v /root/tmp:/tmpdata postgres:17.4 \
    psql -U user1 -d db1 -h database-1.xxxxxx.ap-southeast-1.rds.amazonaws.com

# login_db_me_rds.sh
export PGPASSWORD=user123456
psql -U user1 -d db1 -h database-1.xxxxxx.ap-southeast-1.rds.amazonaws.com


# 非交互式执行 sql

export PGPASSWORD=user123456
psql -h 192.168.0.206 -p 15432 -U user1 -d db1 -f drop_table.sql

echo ${SQL_TXT} | psql -h 192.168.0.206 -p 15432 -U user1 -d db1 -f -

dsn

1
2


dsn: postgres://user1:password@192.168.0.26:5432/db1?sslmode=disable

配置

1
2
3


-- 查看关键配置参数
select name,setting from pg_settings where name in ('max_wal_size','min_wal_size','wal_segment_size','wal_keep_size','wal_level','wal_buffers','fsync');

数据

基础语句

# 不换行输出
\pset pager off

# 不换行输出 方法2/3
export PAGER=cat
psql -c "SELECT * FROM your_table" -P pager=off

# 输出查询
\o output.txt

tables

-- 清空全部表
DO
$$
DECLARE
    table_name text;
BEGIN
    FOR table_name IN (SELECT tablename FROM pg_tables WHERE schemaname = 'public')
    LOOP
        EXECUTE 'TRUNCATE TABLE "' || table_name || '" CASCADE';
    END LOOP;
END
$$
;

-- 删除 all 表, 有删除依赖对象
DO
$$
DECLARE
    table_name text;
BEGIN
    FOR table_name IN (SELECT tablename FROM pg_tables WHERE schemaname = 'public')
    LOOP
        EXECUTE 'DROP TABLE IF EXISTS "' || table_name ||'" CASCADE';
    END LOOP;
END
$$
;

-- 清空表, 并重置自增序列
TRUNCATE TABLE audit_logs CASCADE;
ALTER SEQUENCE audit_logs_id_seq RESTART;


-- 重置序列
DO
$$
DECLARE
    seq_name text;
BEGIN
    FOR seq_name IN (SELECT sequence_name FROM information_schema.sequences WHERE sequence_schema='public')
    LOOP
        EXECUTE 'ALTER SEQUENCE ' || seq_name || ' RESTART';
    END LOOP;
END
$$
;

视图

1
2

-- 查看视图的 创建语句
SELECT pg_get_viewdef('public.my_view'::regclass);

执行计划

1
2

explain sql_line;               -- 执行计划, 但实际没有执行
explain analyze sql_line;       -- 执行计划, 实际有执行

查询sql


-- 按 指定时区进行查询
SELECT COUNT(1) FROM kyc WHERE created_at < ('2024-08-20'::date AT TIME ZONE 'Asia/Shanghai');

-- 查询序列
SELECT nextval('addresses_address_id_seq');

运行巡检

-- 查看数据库大小
SELECT pg_size_pretty(pg_database_size('db1'));


-- 查询所有数据文件对象的大小
SELECT object_type, object_name, schema_name, size FROM
(SELECT
    'Table' AS object_type,
    relname AS object_name,
    schemaname AS schema_name,
    pg_total_relation_size(relid) as size_bytes,
    pg_size_pretty(pg_total_relation_size(relid)) AS size
FROM
    pg_catalog.pg_statio_user_tables
UNION
SELECT
    'Index' AS object_type,
    indexrelname AS object_name,
    schemaname AS schema_name,
    pg_total_relation_size(indexrelid) as size_bytes,
    pg_size_pretty(pg_total_relation_size(indexrelid)) AS size
FROM
    pg_catalog.pg_stat_all_indexes
) as data
ORDER BY
    size_bytes DESC;


--- 只查看表大小
SELECT
    'Table' AS object_type,
    relname AS object_name,
    schemaname AS schema_name,
    pg_total_relation_size(relid) as size_bytes,
    pg_size_pretty(pg_total_relation_size(relid)) AS size
FROM
    pg_catalog.pg_statio_user_tables
ORDER BY
    size_bytes DESC;