dream_653
作者dream_6532019-09-23 15:10
系统应用运维, *****

kubernetes部署-ingress(十一)

字数 21778阅读 3712评论 0赞 2

kubernetes部署-ingress

## 部署ingress

获取ingress的yaml文件

https://github.com/kubernetes/ingress-nginx/tree/nginx-0.20.0/deploy

也可以wget直接下载下来

wget https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/mandatory.yaml  

mandatory.yaml 需要修改images的镜像地址下面的我已经都改好了可以直接用

apiVersion: v1
kind: Namespace
metadata:
  name: ingress-nginx
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: default-http-backend
  labels:
    app.kubernetes.io/name: default-http-backend
    app.kubernetes.io/part-of: ingress-nginx
  namespace: ingress-nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: default-http-backend
      app.kubernetes.io/part-of: ingress-nginx
  template:
    metadata:
      labels:
        app.kubernetes.io/name: default-http-backend
        app.kubernetes.io/part-of: ingress-nginx
    spec:
      terminationGracePeriodSeconds: 60
      containers:
        - name: default-http-backend
          # Any image is permissible as long as:
          # 1. It serves a 404 page at /
          # 2. It serves 200 on a /healthz endpoint
          image: googlecontainer/defaultbackend-amd64:1.5
          livenessProbe:
            httpGet:
              path: /healthz
              port: 8080
              scheme: HTTP
            initialDelaySeconds: 30
            timeoutSeconds: 5
          ports:
            - containerPort: 8080
          resources:
            limits:
              cpu: 10m
              memory: 20Mi
            requests:
              cpu: 10m
              memory: 20Mi

---
apiVersion: v1
kind: Service
metadata:
  name: default-http-backend
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: default-http-backend
    app.kubernetes.io/part-of: ingress-nginx
spec:
  ports:
    - port: 80
      targetPort: 8080
  selector:
    app.kubernetes.io/name: default-http-backend
    app.kubernetes.io/part-of: ingress-nginx

---

kind: ConfigMap
apiVersion: v1
metadata:
  name: nginx-configuration
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---

kind: ConfigMap
apiVersion: v1
metadata:
  name: tcp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---

kind: ConfigMap
apiVersion: v1
metadata:
  name: udp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nginx-ingress-serviceaccount
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: nginx-ingress-clusterrole
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - endpoints
      - nodes
      - pods
      - secrets
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - services
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - "extensions"
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - events
    verbs:
      - create
      - patch
  - apiGroups:
      - "extensions"
    resources:
      - ingresses/status
    verbs:
      - update

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
  name: nginx-ingress-role
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - pods
      - secrets
      - namespaces
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - configmaps
    resourceNames:
      # Defaults to "<election-id>-<ingress-class>"
      # Here: "<ingress-controller-leader>-<nginx>"
      # This has to be adapted if you change either parameter
      # when launching the nginx-ingress-controller.
      - "ingress-controller-leader-nginx"
    verbs:
      - get
      - update
  - apiGroups:
      - ""
    resources:
      - configmaps
    verbs:
      - create
  - apiGroups:
      - ""
    resources:
      - endpoints
    verbs:
      - get

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: nginx-ingress-role-nisa-binding
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: nginx-ingress-role
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: nginx-ingress-clusterrole-nisa-binding
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: nginx-ingress-clusterrole
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---

apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: nginx-ingress-controller
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
  replicas: 2
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
      annotations:
        prometheus.io/port: "10254"
        prometheus.io/scrape: "true"
    spec:
      serviceAccountName: nginx-ingress-serviceaccount
      containers:
        - name: nginx-ingress-controller
          image: siriuszg/nginx-ingress-controller:0.20.0
          args:
            - /nginx-ingress-controller
            - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
            - --configmap=$(POD_NAMESPACE)/nginx-configuration
            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
            - --publish-service=$(POD_NAMESPACE)/ingress-nginx
            - --annotations-prefix=nginx.ingress.kubernetes.io
          securityContext:
            capabilities:
              drop:
                - ALL
              add:
                - NET_BIND_SERVICE
            # www-data -> 33
            runAsUser: 33
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          ports:
            - name: http
              containerPort: 80
            - name: https
              containerPort: 443
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
---

查看pod是否已经起来

kubectl get pod -n ingress-nginx  
NAME READY STATUS RESTARTS AGE  
default-http-backend-cbd6d5994-hzj9l 1/1 Running 0 15h  

这里其实是有问题的从yaml里可以看出会有两个pod

gat deploy查看一下

kubectl get deploy -n ingress-nginx  
NAME READY UP-TO-DATE AVAILABLE AGE  
default-http-backend 1/1 1 1 15h  
nginx-ingress-controller 0/2 0 0 15h  

nginx-ingress-controller这个并没有正确的交给pod来运行

通过以上流程图可以看出deploy把任务交给rs,rs在分别启动pod现在都看不到pod问题应该在rs上
get rs

kubectl get rs -n ingress-nginx  
NAME DESIRED CURRENT READY AGE  
default-http-backend-cbd6d5994 1 1 1 15h  
nginx-ingress-controller-57c8dd86d4 2 0 0 15h  

状态不正确
通过describe查看一下详情

kubectl describe rs/nginx-ingress-controller-57c8dd86d4 -n ingress-nginx  
  
Warning FailedCreate 49m replicaset-controller Error creating: pods "nginx-ingress-controller-57c8dd86d4-97gl6" is forbidden: Secur  ityContext.RunAsUser is forbidden  

通过排查判断是apiserver的参数问题

检查apiserver配置文件: 去除:–enable-admission-plugins中的SecurityContextDeny部分,分别重启apiserver

get一下pod看起来了没

kubectl get pod -n ingress-nginx -o wide  
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS   
GATESdefault-http-backend-cbd6d5994-9bkh8 1/1 Running 0 99s 172.17.82.9 10.167.130.210    
nginx-ingress-controller-57c8dd86d4-p8dz9 1/1 Running 0 97s 172.17.71.8 10.167.130.206    

通过curl来测试是否正常

测试默认返回404页面的主机http-backend

curl http://172.17.82.9   
default backend - 404  

测试直接访问nginx代理看是否会转发到404

curl http://172.17.71.8  
default backend - 404  

更改部署方式

此刻问题来了通过yaml创建的deploy以及server来看好像并没有把nginx端口映射到宿主机上,那么我访问宿主机ip就不会有任何返回,这里可以通过hostport+DaemonSet来解决这个问题
修改yaml文件
1.修改nginx 部署方式为DaemonSet
2.注释replicas: 1
3.增加 hostNetwork: true 在spec: 段内增加
4.增加hostPort 在Ports段内增加


    kubectl delete -f ingress.yaml  #删除重新创建
    kubectl apply -f ingress.yaml

附上修改后的yaml文件内容

apiVersion: v1
kind: Namespace
metadata:
  name: ingress-nginx
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
  name: default-http-backend
  labels:
    app.kubernetes.io/name: default-http-backend
    app.kubernetes.io/part-of: ingress-nginx
  namespace: ingress-nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: default-http-backend
      app.kubernetes.io/part-of: ingress-nginx
  template:
    metadata:
      labels:
        app.kubernetes.io/name: default-http-backend
        app.kubernetes.io/part-of: ingress-nginx
    spec:
      terminationGracePeriodSeconds: 60
      containers:
        - name: default-http-backend
          # Any image is permissible as long as:
          # 1. It serves a 404 page at /
          # 2. It serves 200 on a /healthz endpoint
          image: googlecontainer/defaultbackend-amd64:1.5
          livenessProbe:
            httpGet:
              path: /healthz
              port: 8080
              scheme: HTTP
            initialDelaySeconds: 30
            timeoutSeconds: 5
          ports:
            - containerPort: 8080
          resources:
            limits:
              cpu: 10m
              memory: 20Mi
            requests:
              cpu: 10m
              memory: 20Mi

---
apiVersion: v1
kind: Service
metadata:
  name: default-http-backend
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: default-http-backend
    app.kubernetes.io/part-of: ingress-nginx
spec:
  ports:
    - port: 80
      targetPort: 8080
  selector:
    app.kubernetes.io/name: default-http-backend
    app.kubernetes.io/part-of: ingress-nginx

---

kind: ConfigMap
apiVersion: v1
metadata:
  name: nginx-configuration
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---

kind: ConfigMap
apiVersion: v1
metadata:
  name: tcp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---

kind: ConfigMap
apiVersion: v1
metadata:
  name: udp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nginx-ingress-serviceaccount
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: nginx-ingress-clusterrole
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - endpoints
      - nodes
      - pods
      - secrets
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - services
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - "extensions"
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - events
    verbs:
      - create
      - patch
  - apiGroups:
      - "extensions"
    resources:
      - ingresses/status
    verbs:
      - update

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
  name: nginx-ingress-role
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - pods
      - secrets
      - namespaces
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - configmaps
    resourceNames:
      # Defaults to "<election-id>-<ingress-class>"
      # Here: "<ingress-controller-leader>-<nginx>"
      # This has to be adapted if you change either parameter
      # when launching the nginx-ingress-controller.
      - "ingress-controller-leader-nginx"
    verbs:
      - get
      - update
  - apiGroups:
      - ""
    resources:
      - configmaps
    verbs:
      - create
  - apiGroups:
      - ""
    resources:
      - endpoints
    verbs:
      - get

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: nginx-ingress-role-nisa-binding
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: nginx-ingress-role
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: nginx-ingress-clusterrole-nisa-binding
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: nginx-ingress-clusterrole
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---

apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
  name: nginx-ingress-controller
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
 # replicas: 1
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
      annotations:
        prometheus.io/port: "10254"
        prometheus.io/scrape: "true"
    spec:
      serviceAccountName: nginx-ingress-serviceaccount
      hostNetwork: true
      containers:
        - name: nginx-ingress-controller
          image: siriuszg/nginx-ingress-controller:0.20.0
          args:
            - /nginx-ingress-controller
            - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
            - --configmap=$(POD_NAMESPACE)/nginx-configuration
            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
            - --publish-service=$(POD_NAMESPACE)/ingress-nginx
            - --annotations-prefix=nginx.ingress.kubernetes.io
          securityContext:
            capabilities:
              drop:
                - ALL
              add:
                - NET_BIND_SERVICE
            # www-data -> 33
            runAsUser: 33
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          ports:
            - name: http
              containerPort: 80
              hostPort: 80
            - name: https
              containerPort: 443
              hostPort: 443
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 1
---

ingress部署参考文档
kubernetes ingress nginx

创建一个tomcat并用ingress7层代理转发

创建tomcat服务以及deploy

cat tomcat-ingress.yaml 

apiVersion: v1
kind: Service
metadata:
  name: tomcat
  namespace: default
spec:
  type: ClusterIP
  selector:
    app: tomcat
    release: canary
  ports:
  - name: http
    port: 8080
    targetPort: 8080
---
apiVersion: apps/v1
kind: Deployment
metadata: 
  name: tomcat-deploy
spec:
  replicas: 1
  selector: 
    matchLabels:
      app: tomcat
      release: canary
  template:
    metadata:
      labels:
        app: tomcat
        release: canary
    spec:
      containers:
      - name: tomcat
        image: tomcat:7-alpine
        ports:
        - name: httpd
          containerPort: 8080

查看一下状态

kubectl get pod | grep tomcat  
tomcat-deploy-64b488b68-wk45q 1/1 Running 0 29m  
kubectl get svc | grep tomcat  
tomcat ClusterIP 10.0.0.183  8080/TCP 29m  

创建ingress绑定

cat ingress-tomcat.yaml 
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: ingress-tomcat
  namespace: default
  annotations: 
    kubernets.io/ingress.class: "nginx"
spec:
  rules:
  - host: www.aa.com  #用来解析的域名地址
    http:
      paths:
      - path: 
        backend:
          serviceName: tomcat   #集群服务的名字
          servicePort: 8080        #集群服务开放的端口

访问测试

curl -H "host:www.aa.com" http://10.167.130.206:80 #IP地址为运行ingress-nginx-controller的主机地址,因为只有运行了这个容器才会监听宿主的80端口。  

<!DOCTYPE html>


<html lang="en">
    <head>
        <title>Apache Tomcat/7.0.91</title>

可用命令查看ingress列表

kubectl get ingress
NAME             HOSTS        ADDRESS   PORTS   AGE
ingress-tomcat   www.aa.com             80      34m

kubectl describe ingress ingress-tomcat
Name:             ingress-tomcat
Namespace:        default
Address:          
Default backend:  default-http-backend:80 (<none>)
Rules:
 Host        Path  Backends
 ----        ----  --------
 www.aa.com  
                tomcat:8080 (<none>)

用ingress来负载代理tcp,udp请求

前面已经展示了7层代理,那么4层代理比如mysql对3306端口代理怎么办呢?
ingress也可以做4层代理配置

具体配置如下

创建我们用来测试使用的mysql实例两个

cat mysql.yaml


apiVersion: v1
kind: Service
metadata:
 name: mysql
 namespace: default
spec:
 type: ClusterIP
 selector:
   app: mysql
   release: canary
 ports:
 - name: mysql
   port: 3306
   targetPort: 3306
---
apiVersion: apps/v1
kind: DaemonSet  #每个node都运行一个pod,我就两个node正好用来测试负载效果
metadata: 
 name: mysql-daemonset
spec:
# replicas: 1
 selector: 
   matchLabels:
     app: mysql
     release: canary
 template:
   metadata:
     labels:
       app: mysql
       release: canary
   spec:
     containers:
     - name: mysql
       image: mysql
       env:
         - name: MYSQL_ROOT_PASSWORD  #mysql镜像必须的变量,不写这个变量mysql跑不起来
           value: "mysql"
       ports:
       - name: mysql
         containerPort: 3306
         
         
kubectl apply -f  mysql.yaml #部署mysql pod
kubectl get pod
mysql-daemonset-2xdr7           1/1     Running   0          63m
mysql-daemonset-stvhf           1/1     Running   0          63m

新建一个configmap文件tcp,udp是需要写这个文件来发布服务的

cat configmap.yaml 

kind: ConfigMap
apiVersion: v1
metadata:
  name: tcp-services
  namespace: ingress-nginx
data:
 3306: "default/mysql:3306" #我们的mysql是在默认命名空间里创建的这个自行查看更改
 
 kubectl apply -f configmap.yaml 
 
 [root@node1 ~]# netstat -nlp | grep 3306  #去node节点查看3306端口是否已经监听起来
tcp        0      0 0.0.0.0:3306            0.0.0.0:*               LISTEN      25785/nginx: master 
tcp6       0      0 :::3306     

注:添加或增减configmap直接在这个configmap文件中新增或去除即可,用kubectl apply重新应用,尽量不要直接kubectl delete -f configmap.yaml 因为这样会把整个tcp-services都删掉,删掉后node节点检测不到数据就不会对规则更新,这个和7层代理不太一样,7层可以一个服务创建一个name,4层在创建ingress服务时候就指定tcp-services和udp-services两个文件了,定义位置可以看ingress.yaml的281-282行,还有一种编辑方法

kubectl edit configmap/tcp-services -n ingress-nginx

# Please edit the object below. Lines beginning with a '#' will be ignored,
# and an empty file will abort the edit. If an error occurs while saving this file will be
# reopened with the relevant failures.
#
apiVersion: v1
data:
  "3306": default/mysql:3306  #在下面增加一行就可以,不需要的可以注释或者删除
kind: ConfigMap
metadata:

另外有一点,因为创建完ingress时候Node节点就是监听80和443的,在配置这个mysql时排错过程中发现,3306端口并不会默认监听,只有ingress可以正常连接到mysql集群时,node才会去监听3306端口,有错误可以按照这个思路排错,配置过程中也遇到很多问题,排错思路,容器>容器ip>集群ip>ingress。

断开连接几次试试,应该是轮训算法,分别在两个pod的数据库里写了a和b用来测试负载效果

mysql> show databases; 
+--------------------+
| Database           |
+--------------------+
| a                  |
| information_schema |
| mysql              |
| performance_schema |
| sys                |
+--------------------+

mysql> show databases; 
+--------------------+
| Database           |
+--------------------+
| b                  |
| information_schema |
| mysql              |
| performance_schema |
| sys                |
+--------------------+

如果觉得我的文章对您有用,请点赞。您的支持将鼓励我继续创作!

2

添加新评论0 条评论

Ctrl+Enter 发表

作者其他文章

相关文章

相关问题

相关资料

X社区推广