4 min readFeb 15, 2023
Multus CNI example
Control-plane 10.0.1.100/24
Worker-node 10.0.2.200/24
POD (multtool) — — -POD (GW) — — — Internet
each POD has two networks.
default network is 10.85.0.0/24
data plane network is 10.1.128.0/24
GW POD has fixed IP 10.1.128.2 and fixed MAC.
multtool client POD has default route to 10.1.128.2 to reach internet
default network 10.85.0.0/24 will not install default route on POD (managed in CNI multus config)
Create a net-attach-def for secondary interface
---
apiVersion: "k8s.cni.cncf.io/v1"
kind: NetworkAttachmentDefinition
metadata:
name: br-10-1-128
spec:
config: |-
{
"cniVersion": "0.4.0",
"plugins": [
{
"name": "crio",
"type": "bridge",
"capabilities": { "ips": true },
"bridge": "cni1",
"ipMasq": false,
"ipam": {
"type": "whereabouts",
"range": "10.1.128.0/24",
"gateway": "10.1.128.2",
"log_file": "/tmp/whereabouts.log",
"log_level": "debug",
"routes": [
{
"dst": "10.2.128.0/24",
"gw": "10.1.128.2"
}
],
"exclude": [
"10.1.128.1/32",
"10.1.128.254/32"
]
}
},{
"capabilities": { "mac": true },
"type": "tuning"
}
]
}create a instance as gateway and attach to br-10–1–128 . assign fixed IP and mac to this instance.
---
apiVersion: v1
kind: Service
metadata:
labels:
app: fos
name: fos-deployment
namespace: default
spec:
ports:
- port: 80
protocol: TCP
targetPort: 80
selector:
app: fos
type: ClusterIP
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: fos-deployment
labels:
app: fos
spec:
replicas: 1
selector:
matchLabels:
app: fos
template:
metadata:
labels:
app: fos
annotations:
k8s.v1.cni.cncf.io/networks: '[ { "name": "br-10-1-128" , "ips": [ "10.1.128.2/32" ],"mac": "CA:FE:C0:FF:EE:02" } ]'
spec:
containers:
- name: fos
image: <omitted>:v7231x86
securityContext:
capabilities:
add: ["NET_ADMIN","SYS_ADMIN","NET_RAW"]
ports:
- name: isakmp
containerPort: 500
protocol: UDP
- name: ipsec-nat-t
containerPort: 4500
protocol: UDP
# env:
# - name: MY_VAR
# valueFrom:
# configMapKeyRef:
# name: foscfgfirewallvip
# key: VAR_CLUSTERIPADDRESS
volumeMounts:
- mountPath: /data
name: data-volume
imagePullSecrets:
- name: dockerinterbeing
volumes:
- name: data-volume
persistentVolumeClaim:
claimName: cfosdatacreate a client instance that route traffic to gateway instance.
add an default route to client instance for route traffic to gateway instance.
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: multitool-deployment
labels:
app: multitool
spec:
replicas: 1
selector:
matchLabels:
app: multitool
template:
metadata:
labels:
app: multitool
annotations:
k8s.v1.cni.cncf.io/networks: '[ { "name": "br-10-1-128" , "default-route": ["10.1.128.2"] } ]'
spec:
containers:
- name: multitool
image: praqma/network-multitool
imagePullPolicy: Always
#command: ["/bin/sh","-c"]
args:
- /bin/sh
- -c
- /usr/sbin/nginx -g "daemon off;"
securityContext:
privileged: trueCheck Result
ubuntu@ip-10-0-1-100:~/202301/north-test$ kubectl describe net-attach-def br-10-1-128
Name: br-10-1-128
Namespace: default
Labels: <none>
Annotations: <none>
API Version: k8s.cni.cncf.io/v1
Kind: NetworkAttachmentDefinition
Metadata:
Creation Timestamp: 2023-02-15T04:28:27Z
Generation: 1
Managed Fields:
API Version: k8s.cni.cncf.io/v1
Fields Type: FieldsV1
fieldsV1:
f:metadata:
f:annotations:
.:
f:kubectl.kubernetes.io/last-applied-configuration:
f:spec:
.:
f:config:
Manager: kubectl-client-side-apply
Operation: Update
Time: 2023-02-15T04:28:27Z
Resource Version: 17336
UID: ca07f7fe-d468-408b-9e50-9964ef449a6b
Spec:
Config: {
"cniVersion": "0.4.0",
"plugins": [
{
"name": "crio",
"type": "bridge",
"capabilities": { "ips": true },
"bridge": "cni1",
"ipMasq": false,
"ipam": {
"type": "whereabouts",
"range": "10.1.128.0/24",
"gateway": "10.1.128.2",
"log_file": "/tmp/whereabouts.log",
"log_level": "debug",
"routes": [
{
"dst": "10.2.128.0/24",
"gw": "10.1.128.2"
}
],
"exclude": [
"10.1.128.1/32",
"10.1.128.254/32"
]
}
},{
"capabilities": { "mac": true },
"type": "tuning"
}
]
}
Events: <none>cfos
ubuntu@ip-10-0-1-100:~/202301/north-test$ kubectl exec -it po/fos-deployment-65b475f4bb-mc88r -- sh
# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0@if7: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether aa:b9:2b:ed:f6:3f brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.85.0.74/16 brd 10.85.255.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 1100:200::4a/24 scope global
valid_lft forever preferred_lft forever
inet6 fe80::a8b9:2bff:feed:f63f/64 scope link
valid_lft forever preferred_lft forever
3: net1@if8: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether ca:fe:c0:ff:ee:02 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.1.128.2/24 brd 10.1.128.255 scope global net1
valid_lft forever preferred_lft forever
inet 10.1.128.2/32 scope global net1
valid_lft forever preferred_lft forever
inet6 fe80::c8fe:c0ff:feff:ee02/64 scope link
valid_lft forever preferred_lft forever
# ip r
10.1.128.0/24 dev net1 proto kernel scope link src 10.1.128.2
10.2.128.0/24 via 10.1.128.2 dev net1
10.85.0.0/16 dev eth0 proto kernel scope link src 10.85.0.74
10.96.0.0/12 via 10.85.0.1 dev eth0
# ip r show table 231
default via 10.85.0.1 dev eth0 metric 2
10.1.128.0/24 dev net1 proto kernel scope link src 10.1.128.2
10.2.128.0/24 via 10.1.128.2 dev net1
10.85.0.0/16 dev eth0 proto kernel scope link src 10.85.0.74
10.96.0.0/12 via 10.85.0.1 dev eth0
# ip route get 1.1.1.1
1.1.1.1 via 10.85.0.1 dev eth0 table 231 src 10.85.0.74 uid 0
cacheclient instance
ubuntu@ip-10-0-1-100:~/202301/north-test$ kubectl exec -it po/multitool-deployment-6c974f9fd4-nh45p -- sh
/ # ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
valid_lft forever preferred_lft forever
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0@if4: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 16:ec:e9:ce:30:06 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.85.0.73/16 brd 10.85.255.255 scope global eth0
valid_lft forever preferred_lft forever
inet6 1100:200::49/24 scope global
valid_lft forever preferred_lft forever
inet6 fe80::14ec:e9ff:fece:3006/64 scope link
valid_lft forever preferred_lft forever
3: net1@if6: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default
link/ether 8a:3b:d6:f7:9a:88 brd ff:ff:ff:ff:ff:ff link-netnsid 0
inet 10.1.128.3/24 brd 10.1.128.255 scope global net1
valid_lft forever preferred_lft forever
inet6 fe80::883b:d6ff:fef7:9a88/64 scope link
valid_lft forever preferred_lft forever
/ # ip r
default via 10.1.128.2 dev net1
10.1.128.0/24 dev net1 proto kernel scope link src 10.1.128.3
10.2.128.0/24 via 10.1.128.2 dev net1
10.85.0.0/16 dev eth0 proto kernel scope link src 10.85.0.73
10.96.0.0/12 via 10.85.0.1 dev eth0worker node multus-cni configuration
ubuntu@ip-10-0-2-200:/etc/cni/net.d$ sudo cat 00-multus.conf | yq -P
cniVersion: 0.3.1
name: multus-cni-network
type: multus
kubeconfig: /etc/cni/net.d/multus.d/multus.kubeconfig
delegates:
- cniVersion: 0.3.1
name: crio
type: bridge
bridge: cni0
isGateway: true
ipMasq: true
hairpinMode: true
ipam:
type: host-local
routes:
- dst: 10.96.0.0/12
gw: 10.85.0.1
- dst: 1100:200::1/24
ranges:
- - subnet: 10.85.0.0/16
- - subnet: 1100:200::/24