Kubernetes集群公共服务

云原生 评论

Kubernetes集群核心服务

Kubernetes公共服务

image-20220512190022889

域名解析 DNS

主机IP地址及域名规划

序号 提供服务 IP地址 域名 备注
1 DNS 192.168.10.211
2 Nginx 192.168.10.212 yaml.kubels.com
3 Harbor 192.168.10.213 harbor.kubels.com www.kubels.com
4 NFS 192.168.10.214 nfs.kubels.com

3.2 域名解析 DNS配置

  • bind9
1
[root@localhost ~]# hostnamectl set-hostname dns
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

[root@dns named]# vim /etc/sysconfig/network-scripts/ifcfg-ens33
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens33
UUID=6b19f511-5a6d-4434-bb00-95fc19a62c39
DEVICE=ens33
ONBOOT=yes
DNS1=192.168.77.121  #自己的DNS服务器
DNS2=119.29.29.29
IPADDR=192.168.77.211  
NETMASK=255.255.255.0
GATEWAY=192.168.77.2

1
2
3
4
5
6
7
8
9
10
11
12
13
[root@dns ~]# firewall-cmd --state
not running
[root@dns ~]# echo $?
252
[root@dns ~]# sestatus
SELinux status:                 disabled
[root@dns ~]# echo $?
0

或关闭
[root@dns ~]# systemctl disable firewalld;systemctl stop firewalld

[root@dns ~]# sed -ri.bak 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config

安装dns

dns就是bind9

1
[root@dns ~]# yum -y install bind

修改配置文件的第13行和第21行，添加any: 空格要加上,不要动，只允许自己访问没有意义 

1
2
3
listen-on port 53 { 127.0.0.1; };
修改为
listen-on port 53 { 127.0.0.1;any; };
1
2
3
allow-query     { localhost; };
修改为
allow-query     { localhost;any; };
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
[root@dns ~]# cat -n /etc/named.conf
     1  //
     2  // named.conf
     3  //
     4  // Provided by Red Hat bind package to configure the ISC BIND named(8) DNS
     5  // server as a caching only nameserver (as a localhost DNS resolver only).
     6  //
     7  // See /usr/share/doc/bind*/sample/ for example named configuration files.
     8  //
     9  // See the BIND Administrator's Reference Manual (ARM) for details about the
    10  // configuration located in /usr/share/doc/bind-{version}/Bv9ARM.html
    11
    12  options {
    13          listen-on port 53 { 127.0.0.1;any; };
    14          listen-on-v6 port 53 { ::1; };
    15          directory       "/var/named";
    16          dump-file       "/var/named/data/cache_dump.db";
    17          statistics-file "/var/named/data/named_stats.txt";
    18          memstatistics-file "/var/named/data/named_mem_stats.txt";
    19          recursing-file  "/var/named/data/named.recursing";
    20          secroots-file   "/var/named/data/named.secroots";
    21          allow-query     { localhost;any; };
    22
    23          /*
    24           - If you are building an AUTHORITATIVE DNS server, do NOT enable recursion.
    25           - If you are building a RECURSIVE (caching) DNS server, you need to enable
    26             recursion.
    27           - If your recursive DNS server has a public IP address, you MUST enable access
    28             control to limit queries to your legitimate users. Failing to do so will
    29             cause your server to become part of large scale DNS amplification
    30             attacks. Implementing BCP38 within your network would greatly
    31             reduce such attack surface
    32          */
    33          recursion yes;
    34
    35          dnssec-enable yes;
    36          dnssec-validation yes;
    37
    38          /* Path to ISC DLV key */
    39          bindkeys-file "/etc/named.root.key";
    40
    41          managed-keys-directory "/var/named/dynamic";
    42
    43          pid-file "/run/named/named.pid";
    44          session-keyfile "/run/named/session.key";
    45  };
    46
    47  logging {
    48          channel default_debug {
    49                  file "data/named.run";
    50                  severity dynamic;
    51          };
    52  };
    53
    54  zone "." IN {
    55          type hint;
    56          file "named.ca";
    57  };
    58
    59  include "/etc/named.rfc1912.zones";
    60  include "/etc/named.root.key";

当然域名改自己的主机的hosts文件也是可以的，但是这里不采用这种方式处理

注册域名

可以直接文件里面复制，防止出现，vim里面找到要复制的行，按5 yy 在文件的最后，按一个p键就可以实现复制操作

1
2
3
4
5
6
7
8
9
10
[root@dns ~]# tail -6 /etc/named.rfc1912.zones

[root@dns ~] vim /etc/named.rfc1912.zones

zone "kubels.com" IN {   # 这里的域名随便写
          type master;    # dns可以主备操作，这里只有一个主操作,type选master就可以
          file "kubels.com.zone";   # 这里的文件名称也随便写
          allow-update { none; };   #  不用变   从服务器可以允许更新，主服务器不需要变更
  };

这里的-p 必须要加上 ,复制文件的过程中权限保持不变

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
[root@dns ~]# cd /var/named/
[root@dns named]# ls
data  dynamic  named.ca  named.empty  named.localhost  named.loopback  slaves

[root@dns named]# cp named.localhost  1.conf
[root@dns named]# ll
总用量 20
-rw-r-----. 1 root  root   152 9月  27 23:44 1.conf
drwxrwx---. 2 named named    6 2月  24 2022 data
drwxrwx---. 2 named named    6 2月  24 2022 dynamic
-rw-r-----. 1 root  named 2253 4月   5 2018 named.ca
-rw-r-----. 1 root  named  152 12月 15 2009 named.empty
-rw-r-----. 1 root  named  152 6月  21 2007 named.localhost
-rw-r-----. 1 root  named  168 12月 15 2009 named.loopback
drwxrwx---. 2 named named    6 2月  24 2022 slaves
[root@dns named]# cp -p named.localhost kubels.com.zone
[root@dns named]# ll
总用量 24
-rw-r-----. 1 root  root   152 9月  27 23:44 1.conf
drwxrwx---. 2 named named    6 2月  24 2022 data
drwxrwx---. 2 named named    6 2月  24 2022 dynamic
-rw-r-----. 1 root  named  152 6月  21 2007 kubels.com.zone
-rw-r-----. 1 root  named 2253 4月   5 2018 named.ca
-rw-r-----. 1 root  named  152 12月 15 2009 named.empty
-rw-r-----. 1 root  named  152 6月  21 2007 named.localhost
-rw-r-----. 1 root  named  168 12月 15 2009 named.loopback
drwxrwx---. 2 named named    6 2月  24 2022 slaves
[root@dns named]#

添加DNSzheng

添加域名解析文件，把我们规划的域名最解析，解析之后，我们的网络就可以使用自己的域名了

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
[root@dns named]# vim kubels.com.zone
# 这里修改的是一个域名的正向查询文件。正向查询文件是指把我们的域名解析为具体的ip地址

$TTL 1D  #  设置DNS的有效期为1天
@       IN SOA  kubels.com admin.kubels.com. (     # 第一条是起始域名机构，负责做域名解析   . 必须要加上
                                        0       ; serial
                                        1D      ; refresh
                                        1H      ; retry
                                        1W      ; expire
                                        3H )    ; minimum
@       NS      ns.kubels.com.    # NS 就是域名服务器 起始授权结构本身要注册进来  . 必须要加上
ns      A       192.168.31.211    # 这个就是本身
yaml    A       192.168.31.212
harbor  A       192.168.31.213
www     A       192.168.31.213
nfs     A       192.168.31.214

1
2
[root@dns named]# systemctl enable  named
[root@dns named]# systemctl start named

注意：这里要修改/etc/sysconfig/network-scripts/ifcfg-ens33 修改DNS1 ,一定要把我们自己的DNS嫁到前面去

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
[root@dns named]# vim /etc/sysconfig/network-scripts/ifcfg-ens33
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO=static
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens33
UUID=6b19f511-5a6d-4434-bb00-95fc19a62c39
DEVICE=ens33
ONBOOT=yes
DNS1=192.168.77.121  #自己的DNS服务器
DNS2=119.29.29.29
IPADDR=192.168.77.211  
NETMASK=255.255.255.0
GATEWAY=192.168.77.2
[root@dns named]# systemctl restart network

因为DNS有缓存，所以这里可能更改完成后，不会立即生效

1
systemctl enable --now named

如果还是不生效,执行如下命令再看看效果

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
[root@dns named]# nslookup
> server
Default server: 192.168.77.211
Address: 192.168.77.211#53
Default server: 119.29.29.29
Address: 119.29.29.29#53
> ns.kubels.com
Server:         192.168.77.211
Address:        192.168.77.211#53

Name:   ns.kubels.com
Address: 192.168.77.211
> yaml.kubels.com
Server:         192.168.77.211
Address:        192.168.77.211#53

Name:   yaml.kubels.com
Address: 192.168.77.212
> harbor.kubels.com
Server:         192.168.77.211
Address:        192.168.77.211#53

Name:   harbor.kubels.com
Address: 192.168.77.213
> www.kubels.com
Server:         192.168.77.211
Address:        192.168.77.211#53

Name:   www.kubels.com
Address: 192.168.77.213
> nfs.kubels.com
Server:         192.168.77.211
Address:        192.168.77.211#53

Name:   nfs.kubels.com
Address: 192.168.77.214
> ^C[root@dns named]# ^C
[root@dns named]#


域名解析验证

两种工具

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
[root@dns named]# dig -t a www.baidu.com @19.96.0.10

; <<>> DiG 9.11.4-P2-RedHat-9.11.4-26.P2.el7_9.9 <<>> -t a www.baidu.com @19.96.0.10
;; global options: +cmd
;; Got answer:
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 8322
;; flags: qr rd ra; QUERY: 1, ANSWER: 3, AUTHORITY: 0, ADDITIONAL: 0

;; QUESTION SECTION:
;www.baidu.com.                 IN      A

;; ANSWER SECTION:
www.baidu.com.          1132    IN      CNAME   www.a.shifen.com.
www.a.shifen.com.       167     IN      A       112.80.248.75
www.a.shifen.com.       167     IN      A       112.80.248.76

;; Query time: 5 msec
;; SERVER: 19.96.0.10#53(19.96.0.10)
;; WHEN: 三 9月 28 00:11:20 CST 2022
;; MSG SIZE  rcvd: 90

[root@dns named]#

第二个工具

1
[root@dns named]# yum -y  install bind-utils
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
[root@dns named]# nslookup
> server
Default server: 192.168.10.211
Address: 192.168.10.211#53
Default server: 119.29.29.29
Address: 119.29.29.29#53
> ns.kubels.com
Server:         192.168.10.211
Address:        192.168.10.211#53

Name:   ns.kubels.com
Address: 192.168.10.211

> yaml.kubels.com
Server:         192.168.10.211
Address:        192.168.10.211#53

Name:   yaml.kubels.com
Address: 192.168.10.212

k8s怎么使用呢，修改k8s的主节点和从节点都需要换的dns解析

1
2
3
4
vim /etc/sysconfig/network-scripts/ifcfg-ens33
修改dns1=192.168.31.211
保存
systemctl restart network

YAML资源清单文件托管服务 Nginx

1
[root@localhost ~]# hostnamectl set-hostname nginx
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
[root@nginx ~]# vi /etc/sysconfig/network-scripts/ifcfg-ens33
TYPE=Ethernet
PROXY_METHOD=none
BROWSER_ONLY=no
BOOTPROTO="none"
DEFROUTE=yes
IPV4_FAILURE_FATAL=no
IPV6INIT=yes
IPV6_AUTOCONF=yes
IPV6_DEFROUTE=yes
IPV6_FAILURE_FATAL=no
IPV6_ADDR_GEN_MODE=stable-privacy
NAME=ens33
UUID=6b19f511-5a6d-4434-bb00-95fc19a62c39
DEVICE=ens33
ONBOOT=yes


DNS1=192.168.77.211
DNS2="119.29.29.29"
IPADDR=192.168.77.212
NETMASK=255.255.255.0
GATEWAY=192.168.77.2


1
2
3
4
[root@nginx ~]# firewall-cmd --state
not running
[root@nginx ~]# sestatus
SELinux status:                 disabled

image-20200512094822918

image-20200512094918480

1
[root@nginx ~]# wget http://nginx.org/download/nginx-1.18.0.tar.gz
1
2
[root@nginx ~]# ls soft/
echo-nginx-module-0.61.tar.gz  nginx-1.18.0.tar.gz  ngx-fancyindex-0.4.3.tar.gz
1
[root@nginx ~]# yum -y install gcc prce-devel zlib-devel openssl-devel
1
2
3
4
5
6
7
8
9
[root@nginx ~]# cd soft/
[root@nginx soft]# ls
echo-nginx-module-0.61.tar.gz  nginx-1.18.0.tar.gz  ngx-fancyindex-0.4.3.tar.gz
[root@nginx soft]# tar xf ngx-fancyindex-0.4.3.tar.gz
[root@nginx soft]# tar xf nginx-1.18.0.tar.gz
[root@nginx soft]# tar xf echo-nginx-module-0.61.tar.gz
[root@nginx soft]# ls
echo-nginx-module-0.61         nginx-1.18.0         ngx-fancyindex-0.4.3
echo-nginx-module-0.61.tar.gz  nginx-1.18.0.tar.gz  ngx-fancyindex-0.4.3.tar.gz
1
[root@nginx nginx-1.18.0]# ./configure --prefix=/usr/local/nginx  --with-http_ssl_module --with-http_stub_status_module --with-http_realip_module --add-module=/root/soft/ngx-fancyindex-0.4.3/ --add-module=/root/soft/echo-nginx-module-0.61
1
[root@nginx nginx-1.18.0]# make && make install
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
[root@nginx soft]# cat nginx_manager.sh
#!/usr/bin/bash
# nginx manager

# 定义变量
nginx_base="/usr/local/nginx"
nginxd="$nginx_base/sbin/nginx"
nginx_pid="$nginx_base/logs/nginx.pid"

# 调用系统函数，为输出文字添加颜色
if [ -f /etc/init.d/functions ];then
        source /etc/init.d/functions
fi

# 检测nginx进程是否存在及进程数量是否正常
if [ -f $nginx_pid ];then
        nginx_process_pid=`cat $nginx_pid`
        nginx_process_num=`ps aux | grep "$nginx_process_pid" | wc -l`
fi


# 封装功能

start () {
        if [ -f $nginx_pid ] && [ $nginx_process_num -ge 2 ];then
                echo "nginx already start"
        elif [ ! -f $nginx_pid ] || [ -z "$nginx_process_num" ];then
                action "nginx start" $nginxd
        fi
}

stop () {
        if [ -f $nginx_pid ] && [ $nginx_process_num -ge 2 ];then
                action "nginx stop" kill -s QUIT $nginx_process_pid
        else
                echo "nginx already stop"
        fi
}

status () {
        if [ -f $nginx_pid ] && [ $nginx_process_num -ge 2 ];then
                echo "nginx running"
        else
                echo "nginx stopped"
        fi
}

reload () {
        if [ -f $nginx_pid ] && [ $nginx_process_num -ge 2 ];then
                kill -s HUP $nginx_process_pid
        else
                echo "nginx stopped"
        fi

}

# 调用函数

case $1 in

        start)
                start
                ;;
        stop)
                stop
                ;;
        restart)
                stop
                sleep 1
                start
                ;;
        status)
                status
                ;;
        reload)
                reload
                ;;
        *)
                echo "命令用法: $0 start|stop|restart|status|reload"
esac
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
[root@nginx ~]# cat /usr/local/nginx/conf/nginx.conf

#user  nobody;
worker_processes  1;

#error_log  logs/error.log;
#error_log  logs/error.log  notice;
#error_log  logs/error.log  info;

#pid        logs/nginx.pid;


events {
    worker_connections  1024;
}


http {
    include       mime.types;
    default_type  application/octet-stream;

    #log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
    #                  '$status $body_bytes_sent "$http_referer" '
    #                  '"$http_user_agent" "$http_x_forwarded_for"';

    #access_log  logs/access.log  main;

    sendfile        on;
    #tcp_nopush     on;

    #keepalive_timeout  0;
    keepalive_timeout  65;

    #gzip  on;

    server {
        listen       80;
        server_name  192.168.10.212;

        #charset koi8-r;

        #access_log  logs/host.access.log  main;
        root   html;

        location / {
            fancyindex on;
            fancyindex_exact_size off;
            index  index;
        }

        #error_page  404              /404.html;

        # redirect server error pages to the static page /50x.html
        #
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   html;
        }

        # proxy the PHP scripts to Apache listening on 127.0.0.1:80
        #
        #location ~ \.php$ {
        #    proxy_pass   http://127.0.0.1;
        #}

        # pass the PHP scripts to FastCGI server listening on 127.0.0.1:9000
        #
        #location ~ \.php$ {
        #    root           html;
        #    fastcgi_pass   127.0.0.1:9000;
        #    fastcgi_index  index.php;
        #    fastcgi_param  SCRIPT_FILENAME  /scripts$fastcgi_script_name;
        #    include        fastcgi_params;
        #}

        # deny access to .htaccess files, if Apache's document root
        # concurs with nginx's one
        #
        #location ~ /\.ht {
        #    deny  all;
        #}
    }


    # another virtual host using mix of IP-, name-, and port-based configuration
    #
    #server {
    #    listen       8000;
    #    listen       somename:8080;
    #    server_name  somename  alias  another.alias;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}


    # HTTPS server
    #
    #server {
    #    listen       443 ssl;
    #    server_name  localhost;

    #    ssl_certificate      cert.pem;
    #    ssl_certificate_key  cert.key;

    #    ssl_session_cache    shared:SSL:1m;
    #    ssl_session_timeout  5m;

    #    ssl_ciphers  HIGH:!aNULL:!MD5;
    #    ssl_prefer_server_ciphers  on;

    #    location / {
    #        root   html;
    #        index  index.html index.htm;
    #    }
    #}

}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
[root@nginx ~]# cat /etc/rc.d/rc.local
#!/bin/bash
# THIS FILE IS ADDED FOR COMPATIBILITY PURPOSES
#
# It is highly advisable to create own systemd services or udev rules
# to run scripts during boot instead of using this file.
#
# In contrast to previous versions due to parallel execution during boot
# this script will NOT be run after all other services.
#
# Please note that you must run 'chmod +x /etc/rc.d/rc.local' to ensure
# that this script will be executed during boot.

touch /var/lock/subsys/local
/root/soft/nginx_manager.sh start
1
2
3
4
5
[root@nginx ~]# ll /etc/rc.d/rc.local
-rw-r--r-- 1 root root 507 5月  12 10:27 /etc/rc.d/rc.local
[root@nginx ~]# chmod 744 /etc/rc.d/rc.local
[root@nginx ~]# ll /etc/rc.d/rc.local
-rwxr--r-- 1 root root 507 5月  12 10:27 /etc/rc.d/rc.local

容器镜像仓库 Harbor

主机名及IP地址配置

1
[root@localhost ~]# hostnamectl set-hostname harbor
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
[root@localhost ~]# cat /etc/sysconfig/network-scripts/ifcfg-ens33
TYPE="Ethernet"
PROXY_METHOD="none"
BROWSER_ONLY="no"
BOOTPROTO="static"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="no"
IPV6INIT="yes"
IPV6_AUTOCONF="yes"
IPV6_DEFROUTE="yes"
IPV6_FAILURE_FATAL="no"
IPV6_ADDR_GEN_MODE="stable-privacy"
NAME="eth0"
DEVICE="eth0"
ONBOOT="yes"
IPADDR="192.168.10.213"
PREFIX="24"
GATEWAY="192.168.10.2"
DNS1="192.168.10.211"
DNS2="119.29.29.29"

docker ce安装

Docker安装YUM源准备

使用阿里云开源软件镜像站。

1
# wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo

Docker安装

1
# yum -y install docker-ce

启动Docker服务

1
# systemctl enable --now docker

获取 docker compose二进制文件

1
2
下载docker-compose二进制文件
# wget https://github.com/docker/compose/releases/download/1.25.0/docker-compose-Linux-x86_64
1
2
3
查看已下载二进制文件
# ls
docker-compose-Linux-x86_64
1
2
移动二进制文件到/usr/bin目录，并更名为docker-compose
# mv docker-compose-Linux-x86_64 /usr/bin/docker-compose
1
2
为二进制文件添加可执行权限
# chmod +x /usr/bin/docker-compose
1
2
3
4
5
6
安装完成后，查看docker-compse版本
# docker-compose version
docker-compose version 1.25.0, build 0a186604
docker-py version: 4.1.0
CPython version: 3.7.4
OpenSSL version: OpenSSL 1.1.0l  10 Sep 2019

获取harbor安装文件

1.github上搜索harbor

2.找到goharbor/harbor进入

3.选择合并的tags版本

4.选择离线安装包 harbor-offline-installer-v2.4.1.tgz

image-20220125233739356

1
2
下载harbor离线安装包
# wget https://github.com/goharbor/harbor/releases/download/v2.4.1/harbor-offline-installer-v2.4.1.tgz
1
2
3
查看已下载的离线安装包
# ls
harbor-offline-installer-v2.4.1.tgz

获取TLS文件

1
2
3
查看准备好的证书
# ls
kubels.com_nginx.zip
1
2
3
4
5
6
解压证书压缩包文件
# unzip kubels.com_nginx.zip
Archive:  kubels.com_nginx.zip
Aliyun Certificate Download
  inflating: 6864844_kubels.com.pem
  inflating: 6864844_kubels.com.key
1
2
3
4
查看解压出的文件
# ls
6864844_kubels.com.key
6864844_kubels.com.pem

修改配置文件

1
2
解压harbor离线安装包
# tar xf harbor-offline-installer-v2.4.1.tgz
1
2
3
查看解压出来的目录
# ls
harbor 
1
2
3
4
5
6
移动证书到harbor目录
# # mv 6864844_kubels.com.* harbor

查看harbor目录
# ls harbor
6864844_kubels.com.key  6864844_kubels.com.pem  common.sh  harbor.v2.4.1.tar.gz  harbor.yml.tmpl  install.sh  LICENSE  prepare
1
2
3
创建配置文件
# cd harbor/
# mv harbor.yml.tmpl harbor.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
修改配置文件内容

# vim harbor.yml

# Configuration file of Harbor

# The IP address or hostname to access admin UI and registry service.
# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
hostname: www.kubels.com 修改为域名，而且一定是证书签发的域名

# http related config
http:
  # port for http, default is 80. If https enabled, this port will redirect to https port
  port: 80

# https related config
https:
  # https port for harbor, default is 443
  port: 443
  # The path of cert and key files for nginx
  certificate: /root/harbor/6864844_kubels.com.pem 证书  如果没有证书要注释掉https的这段配置
  private_key: /root/harbor/6864844_kubels.com.key 密钥

# # Uncomment following will enable tls communication between all harbor components
# internal_tls:
#   # set enabled to true means internal tls is enabled
#   enabled: true
#   # put your cert and key files on dir
#   dir: /etc/harbor/tls/internal

# Uncomment external_url if you want to enable external proxy
# And when it enabled the hostname will no longer used
# external_url: https://reg.mydomain.com:8433

# The initial password of Harbor admin
# It only works in first time to install harbor
# Remember Change the admin password from UI after launching Harbor.
harbor_admin_password: 12345 访问密码
......

执行预备脚本

1
2
3
4
prepare base dir is set to /home/harbor
Unable to find image 'goharbor/prepare:v2.6.0' locally
docker: Error response from daemon: Get "https://registry-1.docker.io/v2/": dial tcp: lookup registry-1.docker.io on 192.168.31.211:53: server misbehaving.
See 'docker run --help'.
1
# ./prepare
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
输出
prepare base dir is set to /root/harbor
Clearing the configuration file: /config/portal/nginx.conf
Clearing the configuration file: /config/log/logrotate.conf
Clearing the configuration file: /config/log/rsyslog_docker.conf
Generated configuration file: /config/portal/nginx.conf
Generated configuration file: /config/log/logrotate.conf
Generated configuration file: /config/log/rsyslog_docker.conf
Generated configuration file: /config/nginx/nginx.conf
Generated configuration file: /config/core/env
Generated configuration file: /config/core/app.conf
Generated configuration file: /config/registry/config.yml
Generated configuration file: /config/registryctl/env
Generated configuration file: /config/registryctl/config.yml
Generated configuration file: /config/db/env
Generated configuration file: /config/jobservice/env
Generated configuration file: /config/jobservice/config.yml
Generated and saved secret to file: /data/secret/keys/secretkey
Successfully called func: create_root_cert
Generated configuration file: /compose_location/docker-compose.yml
Clean up the input dir

执行安装脚本

1
# ./install.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
输出
[Step 0]: checking if docker is installed ...

Note: docker version: 20.10.12

[Step 1]: checking docker-compose is installed ...

Note: docker-compose version: 1.25.0

[Step 2]: loading Harbor images ...

[Step 3]: preparing environment ...

[Step 4]: preparing harbor configs ...
prepare base dir is set to /root/harbor

[Step 5]: starting Harbor ...
Creating network "harbor_harbor" with the default driver
Creating harbor-log ... done
Creating harbor-db     ... done
Creating registry      ... done
Creating registryctl   ... done
Creating redis         ... done
Creating harbor-portal ... done
Creating harbor-core   ... done
Creating harbor-jobservice ... done
Creating nginx             ... done
✔ ----Harbor has been installed and started successfully.----

验证运行情况

9个镜像在运行，少一个都不行

1
2
3
4
5
6
7
8
9
10
11
12
# docker ps
CONTAINER ID   IMAGE                                COMMAND                  CREATED              STATUS                        PORTS                                                                            NAMES
71c0db683e4a   goharbor/nginx-photon:v2.4.1         "nginx -g 'daemon of…"   About a minute ago   Up About a minute (healthy)   0.0.0.0:80->8080/tcp, :::80->8080/tcp, 0.0.0.0:443->8443/tcp, :::443->8443/tcp   nginx
4e3b53a86f01   goharbor/harbor-jobservice:v2.4.1    "/harbor/entrypoint.…"   About a minute ago   Up About a minute (healthy)                                                                                    harbor-jobservice
df76e1eabbf7   goharbor/harbor-core:v2.4.1          "/harbor/entrypoint.…"   About a minute ago   Up About a minute (healthy)                                                                                    harbor-core
eeb4d224dfc4   goharbor/harbor-portal:v2.4.1        "nginx -g 'daemon of…"   About a minute ago   Up About a minute (healthy)                                                                                    harbor-portal
70e162c38b59   goharbor/redis-photon:v2.4.1         "redis-server /etc/r…"   About a minute ago   Up About a minute (healthy)                                                                                    redis
8bcc0e9b06ec   goharbor/harbor-registryctl:v2.4.1   "/home/harbor/start.…"   About a minute ago   Up About a minute (healthy)                                                                                    registryctl
d88196398df7   goharbor/registry-photon:v2.4.1      "/home/harbor/entryp…"   About a minute ago   Up About a minute (healthy)                                                                                    registry
ed5ba2ba9c82   goharbor/harbor-db:v2.4.1            "/docker-entrypoint.…"   About a minute ago   Up About a minute (healthy)                                                                                    harbor-db
dcb4b57c7542   goharbor/harbor-log:v2.4.1           "/bin/sh -c /usr/loc…"   About a minute ago   Up About a minute (healthy)   127.0.0.1:1514->10514/tcp                                                        harbor-log

##访问harbor UI界面

在物理机通过浏览器访问

访问102.168.10.213 进入harbor的ui页面

image-20220126000840905

本地docker daemon配置使用本地容器镜像仓库

1
2
3
4
5
[root@harbor ~]# vim /etc/docker/daemon.json
[root@harbor ~]# cat /etc/docker/daemon.json
{
        "insecure-registries": ["http://www.kubels.com"]
}
1
[root@harbor ~]# systemctl restart docker
1
2
3
4
5
6
7
8
[root@harbor ~]# docker login www.kubels.com
Username: admin
Password: 12345
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

Kubernetes集群所有节点配置使用本地容器镜像仓库

1
2
3
4
5
[root@k8s-* ~]# vim /etc/docker/daemon.json
[root@k8s-* ~]# cat /etc/docker/daemon.json
{
        "insecure-registries": ["http://www.kubels.com"]
}
1
[root@k8s-* ~]# systemctl restart docker
1
2
3
4
5
6
7
8
[root@k8s-* ~]# docker login www.kubels.com
Username: admin
Password: 12345
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded

持久化网络文件系统 NFS

1
[root@localhost ~]# hostnamectl set-hostname nfs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
[root@localhost ~]# cat /etc/sysconfig/network-scripts/ifcfg-ens33
TYPE="Ethernet"
PROXY_METHOD="none"
BROWSER_ONLY="no"
BOOTPROTO="static"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="no"
IPV6INIT="yes"
IPV6_AUTOCONF="yes"
IPV6_DEFROUTE="yes"
IPV6_FAILURE_FATAL="no"
IPV6_ADDR_GEN_MODE="stable-privacy"
NAME="eth0"
DEVICE="eth0"
ONBOOT="yes"
IPADDR="192.168.10.214"
PREFIX="24"
GATEWAY="192.168.10.2"
DNS1="192.168.10.211"
DNS2="119.29.29.29"
1
2
3
4
5
[root@nfs ~]# firewall-cmd --state
not running

[root@nfs ~]# sestatus
SELinux status:                 disabled
1
2
3
4
5
6
7
8
9
10
11
[root@nfs ~]# lsblk
NAME                MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT
vda                 252:0    0  100G  0 disk
├─vda1              252:1    0    1G  0 part /boot
└─vda2              252:2    0   99G  0 part
  ├─centos_192-root 253:0    0   50G  0 lvm  /
  ├─centos_192-swap 253:1    0  3.9G  0 lvm  [SWAP]
  └─centos_192-home 253:2    0 45.1G  0 lvm  /home
sdb                 252:16   0  100G  0 disk


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
[root@nfs ~]# mkfs.xfs /dev/sdb
meta-data=/dev/vdb               isize=512    agcount=4, agsize=6553600 blks
         =                       sectsz=512   attr=2, projid32bit=1
         =                       crc=1        finobt=0, sparse=0
data     =                       bsize=4096   blocks=26214400, imaxpct=25
         =                       sunit=0      swidth=0 blks
naming   =version 2              bsize=4096   ascii-ci=0 ftype=1
log      =internal log           bsize=4096   blocks=12800, version=2
         =                       sectsz=512   sunit=0 blks, lazy-count=1
realtime =none                   extsz=4096   blocks=0, rtextents=0
[root@nfs ~]# mkdir /sdb
[root@nfs ~]# echo -e "/dev/sdb\t/sdb\txfs\tdefaults\t0 0" >> /etc/fstab
[root@nfs ~]# tail -1 /etc/fstab
/dev/vdb        /vdb    xfs     defaults 0 0
[root@nfs ~]# mount -a

[root@nfs ~]# df -Th
文件系统                    类型      容量  已用  可用 已用% 挂载点
devtmpfs                    devtmpfs  2.0G     0  2.0G    0% /dev
tmpfs                       tmpfs     2.0G     0  2.0G    0% /dev/shm
tmpfs                       tmpfs     2.0G  8.6M  2.0G    1% /run
tmpfs                       tmpfs     2.0G     0  2.0G    0% /sys/fs/cgroup
/dev/mapper/centos_192-root xfs        50G  2.1G   48G    5% /
/dev/mapper/centos_192-home xfs        46G   33M   46G    1% /home
/dev/vda1                   xfs      1014M  163M  852M   16% /boot
tmpfs                       tmpfs     250M     0  250M    0% /run/user/0
/dev/sdb                    xfs       100G   33M  100G    1% /vdb

1
[root@nfs ~]# yum -y install nfs-utils
1
2
[root@nfs ~]# cat /etc/exports
/sdb *(rw,sync,no_root_squash)
1
2
3
[root@nfs ~]# systemctl enable nfs-server;systemctl start nfs-server
Created symlink from /etc/systemd/system/multi-user.target.wants/nfs-server.service to /usr/lib/systemd/system/nfs-server.service.
[root@nfs ~]# systemctl status nfs-server
1
2
3
[root@nfs ~]# showmount -e
Export list for nfs:
/sdb *
1
2
3
4
5
为了模拟在k8s集群主机中验证nfs可用性
[root@k8s-master01 ~]# yum -y install nfs-utils
[root@k8s-master01 ~]# showmount -e nfs.kubels.com
Export list for nfs.kubels.com:
/sdb *

应用案例

1
2
3
4
5
6
7
8
9
10
创建Pod资源清单文件
[root@nginx ~]# cat 01-create-pod.yaml
apiVersion: v1
kind: Pod
metadata:
  name: pod1
spec:
  containers:
  - name: c1
    image: www.kubels.com/library/nginx:latest
1
[root@nginx ~]# kubectl apply -f http://yaml.kubels.com/01-create-pod.yaml

LsWisdom Blog

宝剑锋从磨砺出，梅花香自苦寒来