docker-bridge-overlay/pkg/plugin/network.go

package plugin

import (
	"context"
	"fmt"
	"math/rand"
	"time"

	container "github.com/docker/docker/api/types/container"
	fTypes "github.com/docker/docker/api/types/filters"
	network "github.com/docker/docker/api/types/network"
	log "github.com/sirupsen/logrus"

	"git.ivasoft.cz/sw/docker-bridge-overlay/pkg/util"
)

const pollTime = 100 * time.Millisecond

// CLIOptionsKey is the key used in create network options by the CLI for custom options
const CLIOptionsKey string = "com.docker.network.generic"

// Implementations of the endpoints described in
// https://github.com/moby/libnetwork/blob/master/docs/remote.md

// CreateNetwork "creates" a new network
func (p *Plugin) CreateNetwork(r CreateNetworkRequest) error {
	log.WithField("options", r.Options).Debug("CreateNetwork options")

	log.WithFields(log.Fields{
		"network": r.NetworkID,
	}).Info("Network created")

	return nil
}

// DeleteNetwork "deletes" a DHCP network (does nothing, the bridge is managed by the user)
func (p *Plugin) DeleteNetwork(r DeleteNetworkRequest) error {
	log.WithField("network", r.NetworkID).Info("Network deleted")
	return nil
}

// CreateEndpoint creates a veth pair and uses udhcpc to acquire an initial IP address on the container end. Docker will
// move the interface into the container's namespace and apply the address.
func (p *Plugin) CreateEndpoint(ctx context.Context, r CreateEndpointRequest) (CreateEndpointResponse, error) {
	log.WithField("options", r.Options).Debug("CreateEndpoint options")
	res := CreateEndpointResponse{
		Interface: &EndpointInterface{},
	}

	if r.Interface != nil && (r.Interface.Address != "" || r.Interface.AddressIPv6 != "") {
		// Address can only be provided by ourselves otherwise it comes from IPAM
		// and the user forgot to set it to null
		return res, util.ErrIPAM
	}

	/*  At this phase we can get only limited info. Especially there is no way
	    to determine the container that is being started.

	    1. The candidate set of containers is all those in the Created status, ie.

		docker ps -a
		CONTAINER ID   IMAGE                             COMMAND   CREATED          STATUS    PORTS     NAMES
		3ec1bfb4df65   alpine                            "/init"   18 seconds ago   Created             test_server.1.5miqq5goljrxmen433v69jy31

	    2. Querying the container being started is not possible because of an underlying lock. However
	    that would only narrow the set of container and is complicated to execute so we do not use it.

		docker inspect <our container> -> HANGS

	    3. Our network does not yet report the endpoint being created so no info there

		docker network inspect <our net> -> no info about the endpoint being created

	    4. We can query the bridge network and get the IP address of the guessed container there
		as the default network is always started first before us.

		docker network inspect bridge
		[
			{
				"Name": "bridge",
				"Id": "eaded6d9ac0097858c501f526e691ca05abe80040ac313a61efb92e1324f3776",
				"Created": "2023-02-15T20:36:24.101135058Z",
				"Scope": "local",
				"Driver": "bridge",
				"EnableIPv6": false,
				"IPAM": {
					"Driver": "default",
					"Options": null,
					"Config": [
						{
							"Subnet": "10.14.1.0/24",
							"Gateway": "10.14.1.1"
						}
					]
				},
				"Internal": false,
				"Attachable": false,
				"Ingress": false,
				"ConfigFrom": {
					"Network": ""
				},
				"ConfigOnly": false,
				"Containers": {
					"3ec1bfb4df65": {
						"Name": "test_server.1.5miqq5goljrxmen433v69jy31",
						"EndpointID": "e4ea43a458468680e92f78c4a6889e8bef2957452ff109a86bdc01d9b604f2b7",
						"MacAddress": "02:42:0a:0e:01:02",
						"IPv4Address": "10.14.1.2/24",
						"IPv6Address": ""
					}
				},
				"Options": {
					"com.docker.network.bridge.default_bridge": "true",
					"com.docker.network.bridge.enable_icc": "true",
					"com.docker.network.bridge.enable_ip_masquerade": "false",
					"com.docker.network.bridge.host_binding_ipv4": "0.0.0.0",
					"com.docker.network.bridge.name": "docker0",
					"com.docker.network.driver.mtu": "1500"
				},
				"Labels": {}
			}
		]
	*/

	// Get all containers in the created status
	createdFilter := fTypes.NewArgs()
	createdFilter.Add("status", "created")
	createdCtrs, err := p.docker.ContainerList(ctx, container.ListOptions{
		Filters: createdFilter,
	})
	if err != nil {
		return res, fmt.Errorf("failed to get the candidate set of containers from Docker: %w", err)
	}

	// Get the bridge network and containers connected to it
	bridgeFilter := fTypes.NewArgs()
	bridgeFilter.Add("name", "bridge")
	bridgeFilter.Add("type", "builtin")
	bridgeNets, err := p.docker.NetworkList(ctx, network.ListOptions{
		Filters: bridgeFilter,
	})
	if err != nil || len(bridgeNets) != 1 {
		return res, fmt.Errorf("failed to get basic bridge info from Docker: %w", err)
	}
	bridgeNet := bridgeNets[0]

	bridgeNet, err = p.docker.NetworkInspect(ctx, bridgeNet.ID, network.InspectOptions{})
	if err != nil {
		return res, fmt.Errorf("failed to get detailed bridge info from Docker: %w", err)
	}

	// Build the final candidate set of containers as intersection of the two collections
	createdOnBridgeCtrs := createdCtrs[:0]
	for _, x := range createdCtrs {
		if _, ok := bridgeNet.Containers[x.ID]; ok {
			createdOnBridgeCtrs = append(createdOnBridgeCtrs, x)
		}
	}
	if len(createdOnBridgeCtrs) == 0 {
		return res, util.ErrNoContainer
	}

	// Guess the container
	// Note: We make a guess here and if we fail the check later in the Join call below then
	//	the container will get re-created by the swarm and it starts all over. Therefore
	//	to eventually succeed we must guess the container randomly.
	rand.Seed(time.Now().UnixNano())
	ctr := createdOnBridgeCtrs[rand.Intn(len(createdOnBridgeCtrs))]

	// Assign the same addresses as the guessed container
	ctrNet := bridgeNet.Containers[ctr.ID]
	p.epToCtrNameGuess[r.EndpointID] = ctrNet.Name
	res.Interface.Address = ctrNet.IPv4Address
	res.Interface.AddressIPv6 = ctrNet.IPv6Address
	res.Interface.MacAddress = ctrNet.MacAddress

	log.WithFields(log.Fields{
		"network":     r.NetworkID[:12],
		"endpoint":    r.EndpointID[:12],
		"mac_address": res.Interface.MacAddress,
		"ip":          res.Interface.Address,
		"ipv6":        res.Interface.AddressIPv6,
	}).Info("Endpoint created")

	return res, nil
}

// EndpointOperInfo retrieves some info about an existing endpoint
func (p *Plugin) EndpointOperInfo(ctx context.Context, r InfoRequest) (InfoResponse, error) {
	res := InfoResponse{}

	log.Info("EndpointOperInfo")

	return res, nil
}

// DeleteEndpoint deletes the veth pair
func (p *Plugin) DeleteEndpoint(r DeleteEndpointRequest) error {
	log.WithFields(log.Fields{
		"network":  r.NetworkID[:12],
		"endpoint": r.EndpointID[:12],
	}).Info("Endpoint deleted")

	delete(p.epToCtrNameGuess, r.EndpointID)

	return nil
}

// Join passes the veth name and route information (gateway from DHCP and existing routes on the host bridge) to Docker
// and starts a persistent DHCP client to maintain the lease on the acquired IP
func (p *Plugin) Join(ctx context.Context, r JoinRequest) (JoinResponse, error) {
	log.WithField("options", r.Options).Debug("Join options")
	res := JoinResponse{}

	/*  At this phase we still get just limited info. However we can determine the container name
		and verify if we guessed correctly in CreateEndpoint

	    1. The candidate set of containers is still all those in the Created status, ie.

		docker ps -a
		CONTAINER ID   IMAGE                             COMMAND   CREATED          STATUS    PORTS     NAMES
		3ec1bfb4df65   alpine                            "/init"   18 seconds ago   Created             test_server.1.5miqq5goljrxmen433v69jy31

	    2. Querying the container being started is not possible because of an underlying lock. However
	    that would only narrow the set of container and is complicated to execute so we do not use it.

		docker inspect <our container> -> HANGS

	    3. Our network does already report the endpoint we created in CreateEndpoint. However it is using
		wrong container identifier (a temporary one prefixed with "ep-") as the container is still not
		tehcnically connected to the sandbox (just shortly after our call finishes).

		docker network inspect test
		[
			{
				"Name": "test",
				"Id": "igjura289na8cii1nj534yn8e",
				"Created": "2023-02-16T19:45:48.590037545Z",
				"Scope": "swarm",
				"Driver": "git.ivasoft.cz/sw/test:latest",
				"EnableIPv6": false,
				"IPAM": {
					"Driver": "null",
					"Options": null,
					"Config": [
						{
							"Subnet": "0.0.0.0/0"
						}
					]
				},
				"Internal": false,
				"Attachable": false,
				"Ingress": false,
				"ConfigFrom": {
					"Network": "testnet"
				},
				"ConfigOnly": false,
				"Containers": {
					"ep-b1f8f067c19684a9f8b5f38e2fd1dd2ebf65ba6e466ea1cfe3d4d84b1aa8c8f1": {
						"Name": "test_server.1.5miqq5goljrxmen433v69jy31",
						"EndpointID": "b1f8f067c19684a9f8b5f38e2fd1dd2ebf65ba6e466ea1cfe3d4d84b1aa8c8f1",
						"MacAddress": "",
						"IPv4Address": "",
						"IPv6Address": ""
					}
				},
				"Options": {},
				"Labels": {},
				"Peers": [
					{
						"Name": "c416d4a38be0",
						"IP": "192.168.14.42"
					},
					{
						"Name": "01c45ee8ae03",
						"IP": "unknown"
					}
				]
			}
		]
	*/

	// Get the network connection of container being started
	thisNet, err := p.docker.NetworkInspect(ctx, r.NetworkID, network.InspectOptions{})
	if err != nil {
		return res, fmt.Errorf("failed to get detailed network info from Docker: %w", err)
	}

	var ep *network.EndpointResource
	for _, i := range thisNet.Containers {
		if i.EndpointID == r.EndpointID {
			ep = &i
			break
		}
	}
	if ep == nil {
		return res, util.ErrNoContainer
	}

	// Validate the guess from the CreateEndpoint
	var ctrName string
	var ok bool
	if ctrName, ok = p.epToCtrNameGuess[r.EndpointID]; ok && ctrName == ep.Name {
		// Guess correct
		delete(p.epToCtrNameGuess, r.EndpointID)
	} else {
		// Guess failed, the container must be re-created
		return res, util.ErrGuessFailed
	}

	log.WithFields(log.Fields{
		"network":       r.NetworkID[:12],
		"endpoint":      r.EndpointID[:12],
		"sandbox":       r.SandboxKey,
		"containerName": ctrName,
	}).Info("Joined sandbox to endpoint")

	return res, nil
}

// Leave stops the persistent DHCP client for an endpoint
func (p *Plugin) Leave(ctx context.Context, r LeaveRequest) error {

	log.WithFields(log.Fields{
		"network":  r.NetworkID[:12],
		"endpoint": r.EndpointID[:12],
	}).Info("Sandbox left endpoint")

	return nil
}