package plugin import ( "context" "fmt" "math/rand" "time" dTypes "github.com/docker/docker/api/types" fTypes "github.com/docker/docker/api/types/filters" log "github.com/sirupsen/logrus" "git.ivasoft.cz/sw/docker-bridge-overlay/pkg/util" ) const pollTime = 100 * time.Millisecond // CLIOptionsKey is the key used in create network options by the CLI for custom options const CLIOptionsKey string = "com.docker.network.generic" // Implementations of the endpoints described in // https://github.com/moby/libnetwork/blob/master/docs/remote.md // CreateNetwork "creates" a new network func (p *Plugin) CreateNetwork(r CreateNetworkRequest) error { log.WithField("options", r.Options).Debug("CreateNetwork options") log.WithFields(log.Fields{ "network": r.NetworkID, }).Info("Network created") return nil } // DeleteNetwork "deletes" a DHCP network (does nothing, the bridge is managed by the user) func (p *Plugin) DeleteNetwork(r DeleteNetworkRequest) error { log.WithField("network", r.NetworkID).Info("Network deleted") return nil } // CreateEndpoint creates a veth pair and uses udhcpc to acquire an initial IP address on the container end. Docker will // move the interface into the container's namespace and apply the address. func (p *Plugin) CreateEndpoint(ctx context.Context, r CreateEndpointRequest) (CreateEndpointResponse, error) { log.WithField("options", r.Options).Debug("CreateEndpoint options") res := CreateEndpointResponse{ Interface: &EndpointInterface{}, } if r.Interface != nil && (r.Interface.Address != "" || r.Interface.AddressIPv6 != "") { // Address can only be provided by ourselves otherwise it comes from IPAM // and the user forgot to set it to null return res, util.ErrIPAM } /* At this phase we can get only limited info. Especially there is no way to determine the container that is being started. 1. The candidate set of containers is all those in the Created status, ie. docker ps -a CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 3ec1bfb4df65 alpine "/init" 18 seconds ago Created test_server.1.5miqq5goljrxmen433v69jy31 2. Querying the container being started is not possible because of an underlying lock. However that would only narrow the set of container and is complicated to execute so we do not use it. docker inspect -> HANGS 3. Our network does not yet report the endpoint being created so no info there docker network inspect -> no info about the endpoint being created 4. We can query the bridge network and get the IP address of the guessed container there as the default network is always started first before us. docker network inspect bridge [ { "Name": "bridge", "Id": "eaded6d9ac0097858c501f526e691ca05abe80040ac313a61efb92e1324f3776", "Created": "2023-02-15T20:36:24.101135058Z", "Scope": "local", "Driver": "bridge", "EnableIPv6": false, "IPAM": { "Driver": "default", "Options": null, "Config": [ { "Subnet": "10.14.1.0/24", "Gateway": "10.14.1.1" } ] }, "Internal": false, "Attachable": false, "Ingress": false, "ConfigFrom": { "Network": "" }, "ConfigOnly": false, "Containers": { "3ec1bfb4df65": { "Name": "test_server.1.5miqq5goljrxmen433v69jy31", "EndpointID": "e4ea43a458468680e92f78c4a6889e8bef2957452ff109a86bdc01d9b604f2b7", "MacAddress": "02:42:0a:0e:01:02", "IPv4Address": "10.14.1.2/24", "IPv6Address": "" } }, "Options": { "com.docker.network.bridge.default_bridge": "true", "com.docker.network.bridge.enable_icc": "true", "com.docker.network.bridge.enable_ip_masquerade": "false", "com.docker.network.bridge.host_binding_ipv4": "0.0.0.0", "com.docker.network.bridge.name": "docker0", "com.docker.network.driver.mtu": "1500" }, "Labels": {} } ] */ // Get all containers in the created status createdFilter := fTypes.NewArgs() createdFilter.Add("status", "created") createdCtrs, err := p.docker.ContainerList(ctx, dTypes.ContainerListOptions{ Filters: createdFilter, }) if err != nil { return res, fmt.Errorf("failed to get the candidate set of containers from Docker: %w", err) } // Get the bridge network and containers connected to it bridgeFilter := fTypes.NewArgs() bridgeFilter.Add("name", "bridge") bridgeFilter.Add("type", "builtin") bridgeNets, err := p.docker.NetworkList(ctx, dTypes.NetworkListOptions{ Filters: bridgeFilter, }) if err != nil || len(bridgeNets) != 1 { return res, fmt.Errorf("failed to get basic bridge info from Docker: %w", err) } bridgeNet := bridgeNets[0] bridgeNet, err = p.docker.NetworkInspect(ctx, bridgeNet.ID, dTypes.NetworkInspectOptions{}) if err != nil { return res, fmt.Errorf("failed to get detailed bridge info from Docker: %w", err) } // Build the final candidate set of containers as intersection of the two collections createdOnBridgeCtrs := createdCtrs[:0] for _, x := range createdCtrs { if _, ok := bridgeNet.Containers[x.ID]; ok { createdOnBridgeCtrs = append(createdOnBridgeCtrs, x) } } if len(createdOnBridgeCtrs) == 0 { return res, util.ErrNoContainer } // Guess the container // Note: We make a guess here and if we fail the check later in the Join call below then // the container will get re-created by the swarm and it starts all over. Therefore // to eventually succeed we must guess the container randomly. rand.Seed(time.Now().UnixNano()) ctr := createdOnBridgeCtrs[rand.Intn(len(createdOnBridgeCtrs))] // Assign the same addresses as the guessed container ctrNet := bridgeNet.Containers[ctr.ID] p.epToCtrNameGuess[r.EndpointID] = ctrNet.Name res.Interface.Address = ctrNet.IPv4Address res.Interface.AddressIPv6 = ctrNet.IPv6Address res.Interface.MacAddress = ctrNet.MacAddress log.WithFields(log.Fields{ "network": r.NetworkID[:12], "endpoint": r.EndpointID[:12], "mac_address": res.Interface.MacAddress, "ip": res.Interface.Address, "ipv6": res.Interface.AddressIPv6, }).Info("Endpoint created") return res, nil } // EndpointOperInfo retrieves some info about an existing endpoint func (p *Plugin) EndpointOperInfo(ctx context.Context, r InfoRequest) (InfoResponse, error) { res := InfoResponse{} log.Info("EndpointOperInfo") return res, nil } // DeleteEndpoint deletes the veth pair func (p *Plugin) DeleteEndpoint(r DeleteEndpointRequest) error { log.WithFields(log.Fields{ "network": r.NetworkID[:12], "endpoint": r.EndpointID[:12], }).Info("Endpoint deleted") delete(p.epToCtrNameGuess, r.EndpointID) return nil } // Join passes the veth name and route information (gateway from DHCP and existing routes on the host bridge) to Docker // and starts a persistent DHCP client to maintain the lease on the acquired IP func (p *Plugin) Join(ctx context.Context, r JoinRequest) (JoinResponse, error) { log.WithField("options", r.Options).Debug("Join options") res := JoinResponse{} /* At this phase we still get just limited info. However we can determine the container name and verify if we guessed correctly in CreateEndpoint 1. The candidate set of containers is still all those in the Created status, ie. docker ps -a CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 3ec1bfb4df65 alpine "/init" 18 seconds ago Created test_server.1.5miqq5goljrxmen433v69jy31 2. Querying the container being started is not possible because of an underlying lock. However that would only narrow the set of container and is complicated to execute so we do not use it. docker inspect -> HANGS 3. Our network does already report the endpoint we created in CreateEndpoint. However it is using wrong container identifier (a temporary one prefixed with "ep-") as the container is still not tehcnically connected to the sandbox (just shortly after our call finishes). docker network inspect test [ { "Name": "test", "Id": "igjura289na8cii1nj534yn8e", "Created": "2023-02-16T19:45:48.590037545Z", "Scope": "swarm", "Driver": "git.ivasoft.cz/sw/test:latest", "EnableIPv6": false, "IPAM": { "Driver": "null", "Options": null, "Config": [ { "Subnet": "0.0.0.0/0" } ] }, "Internal": false, "Attachable": false, "Ingress": false, "ConfigFrom": { "Network": "testnet" }, "ConfigOnly": false, "Containers": { "ep-b1f8f067c19684a9f8b5f38e2fd1dd2ebf65ba6e466ea1cfe3d4d84b1aa8c8f1": { "Name": "test_server.1.5miqq5goljrxmen433v69jy31", "EndpointID": "b1f8f067c19684a9f8b5f38e2fd1dd2ebf65ba6e466ea1cfe3d4d84b1aa8c8f1", "MacAddress": "", "IPv4Address": "", "IPv6Address": "" } }, "Options": {}, "Labels": {}, "Peers": [ { "Name": "c416d4a38be0", "IP": "192.168.14.42" }, { "Name": "01c45ee8ae03", "IP": "unknown" } ] } ] */ // Get the network connection of container being started thisNet, err := p.docker.NetworkInspect(ctx, r.NetworkID, dTypes.NetworkInspectOptions{}) if err != nil { return res, fmt.Errorf("failed to get detailed network info from Docker: %w", err) } var ep *dTypes.EndpointResource for _, i := range thisNet.Containers { if i.EndpointID == r.EndpointID { ep = &i break } } if ep == nil { return res, util.ErrNoContainer } // Validate the guess from the CreateEndpoint var ctrName string var ok bool if ctrName, ok = p.epToCtrNameGuess[r.EndpointID]; ok && ctrName == ep.Name { // Guess correct delete(p.epToCtrNameGuess, r.EndpointID) } else { // Guess failed, the container must be re-created return res, util.ErrGuessFailed } log.WithFields(log.Fields{ "network": r.NetworkID[:12], "endpoint": r.EndpointID[:12], "sandbox": r.SandboxKey, "containerName": ctrName, }).Info("Joined sandbox to endpoint") return res, nil } // Leave stops the persistent DHCP client for an endpoint func (p *Plugin) Leave(ctx context.Context, r LeaveRequest) error { log.WithFields(log.Fields{ "network": r.NetworkID[:12], "endpoint": r.EndpointID[:12], }).Info("Sandbox left endpoint") return nil }