「六」DOCKER源码分析6 网络部分执行流分析（libnetwork源码解读）

func (daemon *Daemon) initNetworkController(config *config.Config, activeSandboxes map[string]interface{}) (libnetwork.NetworkController, error) {
	//network options,如OptionExecRoot、OptionDefaultDriver、OptionDefaultNetwork等
	netOptions, err := daemon.networkOptions(config, daemon.PluginStore, activeSandboxes)
	//根据netOptions创建net controller，2.3.1分析
	controller, err := libnetwork.New(netOptions...)
	// Initialize default network on "null"
	if n, _ := controller.NetworkByName("none"); n == nil {
		//如果没有就新建一个，NewNetwork()在2.3.2分析
		if _, err := controller.NewNetwork("null", "none", "", libnetwork.NetworkOptionPersist(true)); err != nil {
			return nil, fmt.Errorf("Error creating default \"null\" network: %v", err)
		}
	}
	// Initialize default network on "host"
	if n, _ := controller.NetworkByName("host"); n == nil {
		if _, err := controller.NewNetwork("host", "host", "", libnetwork.NetworkOptionPersist(true)); err != nil {
			return nil, fmt.Errorf("Error creating default \"host\" network: %v", err)
		}
	}
	// Clear stale bridge network
	if n, err := controller.NetworkByName("bridge"); err == nil {
		if err = n.Delete(); err != nil {
			return nil, fmt.Errorf("could not delete the default bridge network: %v", err)
		}
	}
	if !config.DisableBridge {
		// Initialize default driver "bridge"，在2.3.3分析
		if err := initBridgeDriver(controller, config); err != nil {
			return nil, err
		}
	} else {
		removeDefaultBridgeInterface()
	}
	return controller, nil
}

func New(cfgOptions ...config.Option) (NetworkController, error) {
	c := &controller{
		id:              stringid.GenerateRandomID(),
		cfg:             config.ParseConfigOptions(cfgOptions...),
		sandboxes:       sandboxTable{},
		svcRecords:      make(map[string]svcInfo),
		serviceBindings: make(map[serviceKey]*service),
		agentInitDone:   make(chan struct{}),
		networkLocker:   locker.New(),
	}
	c.initStores()
	//DrvRegistry holds the registry of all network drivers and IPAM drivers that it knows about.
	drvRegistry, err := drvregistry.New(c.getStore(datastore.LocalScope), c.getStore(datastore.GlobalScope), c.RegisterDriver, nil, c.cfg.PluginGetter)
	/*the daemon are enabled for the experimental features，这是一个bool变量，true的话会增加额外的驱动初始化结构体，如ipvlan
	这个函数调用的是./libnetwork/drivers_linux.go，返回的是bridge、host、macvlan、null、remote、overlay的初始化结构体initializer数组，这个结构体包含了驱动的初始化函数和驱动的名字,以bridge驱动为例：{bridge.Init, "bridge"}
	在2.3.1.1中分析bridge的初始化过程
	*/
	for _, i := range getInitializers(c.cfg.Daemon.Experimental) {
		var dcfg map[string]interface{}

		// External plugins don't need config passed through daemon. They can
		// bootstrap themselves
		//如果不是remote驱动，则创建driver config
		if i.ntype != "remote" {
			dcfg = c.makeDriverConfig(i.ntype)
		}
		//将这些驱动的名字、初始化函数、config添加到drvRegistry中
		if err := drvRegistry.AddDriver(i.ntype, i.fn, dcfg); err != nil {
			return nil, err
		}
	}
	//初始化IPAMDrivers
	initIPAMDrivers(drvRegistry, nil, c.getStore(datastore.GlobalScope))
	//赋值controller的drvRegistry
	c.drvRegistry = drvRegistry

	if c.cfg != nil && c.cfg.Cluster.Watcher != nil {
		if err := c.initDiscovery(c.cfg.Cluster.Watcher); err != nil {
			// Failing to initialize discovery is a bad situation to be in.
			// But it cannot fail creating the Controller
			logrus.Errorf("Failed to Initialize Discovery : %v", err)
		}
	}
	//遍历controller所有network，如果network has special drivers, which do not need to perform any network plumbing,将添加到controller中
	c.WalkNetworks(populateSpecial)
	/*
	Reserve pools first before doing cleanup. Otherwise the cleanups of endpoint/network and sandbox below will generate many unnecessary warnings
	*/
	c.reservePools()
	// Cleanup resources
	c.sandboxCleanup(c.cfg.ActiveSandboxes)
	c.cleanupLocalEndpoints()
	c.networkCleanup()
	// /run/docker/libnetwork/container ID.sock开始监听，2.3.1.2分析
	c.startExternalKeyListener()
	return c, nil
}

func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
	//在下面详细看
	d := newDriver()
	if err := d.configure(config); err != nil {
		return err
	}
	// Capability represents the high level capabilities of the drivers which libnetwork can make use of
	c := driverapi.Capability{
		DataScope: datastore.LocalScope,
	}
	//RegisterDriver provides a way for Remote drivers to dynamically register new NetworkType and associate with a driver instance
	return dc.RegisterDriver(networkType, d, c)
}

func newDriver() *driver {
	//返回一个driver结构体
	return &driver{networks: map[string]*bridgeNetwork{}, config: &configuration{}}
}

type driver struct {
	// configuration info for the "bridge" driver.
	config         *configuration
	network        *bridgeNetwork
	// ChainInfo defines the iptables chain.
	natChain       *iptables.ChainInfo
	filterChain    *iptables.ChainInfo
	isolationChain *iptables.ChainInfo
	networks       map[string]*bridgeNetwork
	store          datastore.DataStore
	/*
	Handle is an handle for the netlink requests on a specific network namespace. All the requests on the same netlink family share the same netlink socket,which gets released when the handle is deleted.
	*/
	nlh            *netlink.Handle
	sync.Mutex
}

func (c *controller) startExternalKeyListener() error {
	//udsBase = "/run/docker/libnetwork/"，这里创建"/run/docker/libnetwork/"目录，访问权限0600
	os.MkdirAll(udsBase, 0600)
	//uds="/run/docker/libnetwork/"+c.id+".sock"
	uds := udsBase + c.id + ".sock"
	//监听这个sock
	l, err := net.Listen("unix", uds)
	os.Chmod(uds, 0600)
	c.Lock()
	c.extKeyListener = l
	c.Unlock()
	//新起一个routine，接收客户端连接
	go c.acceptClientConnections(uds, l)
	return nil
}

controller.NewNetwork("bridge", "bridge", "",
		libnetwork.NetworkOptionEnableIPv6(config.BridgeConfig.EnableIPv6),
		libnetwork.NetworkOptionDriverOpts(netOption),
		libnetwork.NetworkOptionIpam("default", "", v4Conf, v6Conf, nil),
		libnetwork.NetworkOptionDeferIPv6Alloc(deferIPv6Alloc))

func (c *controller) NewNetwork(networkType, name string, id string, options ...NetworkOption) (Network, error) {
	//对于例子中，id=="",这里是查看对应id的network是否已经存在，如果已经存在则返回错误
	if id != "" {
		c.networkLocker.Lock(id)
		defer c.networkLocker.Unlock(id)

		if _, err := c.NetworkByID(id); err == nil {
			return nil, NetworkNameError(id)
		}
	}
	//检验network name
	if !config.IsValidName(name) {
		return nil, ErrInvalidName(name)
	}
	//随机生成一个network id
	if id == "" {
		id = stringid.GenerateRandomID()
	}
	// 返回的是 "default"，是default ipam driver的名字
	defaultIpam := defaultIpamForNetworkType(networkType)
	//创建network结构体
	network := &network{
		name:        name,
		networkType: networkType,
		generic:     map[string]interface{}{netlabel.GenericData: make(map[string]string)},
		ipamType:    defaultIpam,
		id:          id,
		created:     time.Now(),
		ctrlr:       c,
		persist:     true,
		drvOnce:     &sync.Once{},
	}
	//network的options挨个进行调用
	network.processOptions(options...)
	//根据网络类型，获得驱动以及驱动的capability，true表示如果没在找到这种类型的network的驱动，则进行装载
	_, cap, err := network.resolveDriver(networkType, true)
	//如果network的ingress为true，cap的DataScope不为"global"
	if network.ingress && cap.DataScope != datastore.GlobalScope {
		//返回错误，Ingress network只能是global scope network
		return nil, types.ForbiddenErrorf("Ingress network can only be global scope network")
	}
	// cap的DataScope为"global"，controller是manager和agent，network的dynamic为false
	if cap.DataScope == datastore.GlobalScope && !c.isDistributedControl() && !network.dynamic {
		if c.isManager() {
			// For non-distributed controlled environment, globalscoped non-dynamic networks are redirected to Manager
			return nil, ManagerRedirectError(name)
		}
		//不能在worker节点创建multi-host网络
		return nil, types.ForbiddenErrorf("Cannot create a multi-host network from a worker node. Please create the network from a manager node.")
	}
	//在分配各种资源前，确保这种类型的网络驱动是可用的
	if _, err := network.driver(true); err != nil {
		return nil, err
	}
	//分配ipam
	network.ipamAllocate()
	//创建network，下面继续分析
	c.addNetwork(network)
	/*
	First store the endpoint count, then the network.
	To avoid to end up with a datastore containing a network and not an epCnt,in case of an ungraceful shutdown during this function call.
	*/
	epCnt := &endpointCnt{n: network}
	if err = c.updateToStore(epCnt); err != nil {
		return nil, err
	}
	//network的endpoint count
	network.epCnt = epCnt
	//将network更新存储到store中
	c.updateToStore(network)
	//join network into agent cluster
	joinCluster(network)
	if !c.isDistributedControl() {
		c.Lock()
		/*
		In the filter table FORWARD chain first rule should be to jump to INGRESS-CHAIN.
		This chain has the rules to allow access to the published ports for swarm tasks from local bridge networks and docker_gwbridge (ie:taks on other swarm netwroks)
		*/
		arrangeIngressFilterRule()
		c.Unlock()
	}
	return network, nil
}

func (c *controller) addNetwork(n *network) error {
	//获得对应network的driver，如果不存在则载入
	d, err := n.driver(true)
	// Create the network，下面接着分析
	d.CreateNetwork(n.id, n.generic, n, n.getIPData(4), n.getIPData(6))
	//在linux中是个空函数，Stub implementations for DNS related functions
	n.startResolver()
	return nil
}

func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
	if len(ipV4Data) == 0 || ipV4Data[0].Pool.String() == "0.0.0.0/0" {
		return types.BadRequestErrorf("ipv4 pool is empty")
	}
	// Sanity checks
	d.Lock()
	if _, ok := d.networks[id]; ok {
		d.Unlock()
		//这个network id已经存在
		return types.ForbiddenErrorf("network %s exists", id)
	}
	d.Unlock()
	// Parse and validate the config. It should not be conflict with existing networks' config,config为networkConfiguration类型
	config, err := parseNetworkOptions(id, option)
	//对ip地址的处理
	config.processIPAM(id, ipV4Data, ipV6Data)
	//下面接着分析
	d.createNetwork(config)
	//更新config到store中
	return d.storeUpdate(config)
}

func (d *driver) createNetwork(config *networkConfiguration) error {
	//initializes OS context while configuring network resources
	defer osl.InitOSContext()()
	//把d.networks转成slice
	networkList := d.getNetworks()
	
	for i, nw := range networkList {
		nw.Lock()
		nwConfig := nw.config
		nw.Unlock()
		//Conflicts check if two NetworkConfiguration objects overlap,查看新建的network config是否与之前的network冲突
		if err := nwConfig.Conflicts(config); err != nil {
			//对于新建network config冲突的处理
			//如果是使用的defaultBridge
			if config.DefaultBridge {
				// We encountered and identified a stale default network
				// We must delete it as libnetwork is the source of thruth
				// The default network being created must be the only one
				// This can happen only from docker 1.12 on ward
				logrus.Infof("Removing stale default bridge network %s (%s)", nwConfig.ID, nwConfig.BridgeName)
				if err := d.DeleteNetwork(nwConfig.ID); err != nil {
					logrus.Warnf("Failed to remove stale default network: %s (%s): %v. Will remove from store.", nwConfig.ID, nwConfig.BridgeName, err)
					d.storeDelete(nwConfig)
				}
				networkList = append(networkList[:i], networkList[i+1:]...)
			} else {
				return types.ForbiddenErrorf("cannot create network %s (%s): conflicts with network %s (%s): %s",
					config.ID, config.BridgeName, nwConfig.ID, nwConfig.BridgeName, err.Error())
			}
		}
	}
	// Create and set network handler in driver
	network := &bridgeNetwork{
		id:         config.ID,
		endpoints:  make(map[string]*bridgeEndpoint),
		config:     config,
		portMapper: portmapper.New(d.config.UserlandProxyPath),
		driver:     d,
	}
	d.Lock()
	//新建的network存入driver
	d.networks[config.ID] = network
	d.Unlock()
	// Initialize handle when needed
	d.Lock()
	if d.nlh == nil {
		// NlHandle returns the netlink handler,*netlink.Handle
		d.nlh = ns.NlHandle()
	}
	d.Unlock()
	/*
	Create or retrieve the bridge L3 interface
	newInterface creates a new bridge interface structure. It attempts to find an already existing device identified by the configuration BridgeName field, or the default bridge name when unspecified, but doesn't attempt to create one when missing
	下面接着分析
	*/
	bridgeIface, err := newInterface(d.nlh, config)
	network.bridge = bridgeIface
	/*
	Verify the network configuration does not conflict with previously installed networks. This step is needed now because driver might have now set the bridge name on this config struct. And because we need to check for possible address conflicts, so we need to check against operationa lnetworks.
	*/
	config.conflictsWithNetworks(config.ID, networkList)

	setupNetworkIsolationRules := func(config *networkConfiguration, i *bridgeInterface) error {
		/*
		isolateNetwork():Install/Removes the iptables rules needed to isolate this network from each of the other networks.
		true是install，false是removes
		*/
		if err := network.isolateNetwork(networkList, true); err != nil {
			if err := network.isolateNetwork(networkList, false); err != nil {
				logrus.Warnf("Failed on removing the inter-network iptables rules on cleanup: %v", err)
			}
			return err
		}
		//注册Iptables清理函数
		network.registerIptCleanFunc(func() error {
			nwList := d.getNetworks()
			//清理driver中所有的network的iptables
			return network.isolateNetwork(nwList, false)
		})
		return nil
	}
	// Prepare the bridge setup configuration， bridgeSetup结构体
	bridgeSetup := newBridgeSetup(config, bridgeIface)
	/*
	If the bridge interface doesn't exist, we need to start the setup steps by creating a new device and assigning it an IPv4 address.
	*/
	bridgeAlreadyExists := bridgeIface.exists()
	if !bridgeAlreadyExists {
		// SetupDevice create a new bridge interface，创建一个link并分配mac地址，下面接着分析
		bridgeSetup.queueStep(setupDevice)
	}
	// Even if a bridge exists try to setup IPv4.给bridge分配ipv4和gateway ipv4地址
	bridgeSetup.queueStep(setupBridgeIPv4)
	//是否允许ipv6转发
	enableIPv6Forwarding := d.config.EnableIPForwarding && config.AddressIPv6 != nil
	// Conditionally queue setup steps depending on configuration values.
	//依次将下面的setupstep加入到bridgeSetup.queueStep中
	for _, step := range []struct {
		Condition bool
		Fn        setupStep
	}{
		// Enable IPv6 on the bridge if required. We do this even for a
		// previously  existing bridge, as it may be here from a previous
		// installation where IPv6 wasn't supported yet and needs to be
		// assigned an IPv6 link-local address.
		{config.EnableIPv6, setupBridgeIPv6},

		// We ensure that the bridge has the expectedIPv4 and IPv6 addresses in
		// the case of a previously existing device.
		{bridgeAlreadyExists, setupVerifyAndReconcile},

		// Enable IPv6 Forwarding
		{enableIPv6Forwarding, setupIPv6Forwarding},

		// Setup Loopback Adresses Routing
		{!d.config.EnableUserlandProxy, setupLoopbackAdressesRouting},

		// Setup IPTables.
		{d.config.EnableIPTables, network.setupIPTables},

		//We want to track firewalld configuration so that
		//if it is started/reloaded, the rules can be applied correctly
		{d.config.EnableIPTables, network.setupFirewalld},

		// Setup DefaultGatewayIPv4
		{config.DefaultGatewayIPv4 != nil, setupGatewayIPv4},

		// Setup DefaultGatewayIPv6
		{config.DefaultGatewayIPv6 != nil, setupGatewayIPv6},

		// Add inter-network communication rules.
		{d.config.EnableIPTables, setupNetworkIsolationRules},

		//Configure bridge networking filtering if ICC is off and IP tables are enabled
		{!config.EnableICC && d.config.EnableIPTables, setupBridgeNetFiltering},
	} {
		if step.Condition {
			bridgeSetup.queueStep(step.Fn)
		}
	}
	// Apply the prepared list of steps, and abort at the first error. 
	//SetupDeviceUp ups the given bridge interface.
	bridgeSetup.queueStep(setupDeviceUp)
	//运行上面所有的setupsteps
	bridgeSetup.apply()
	return nil
}

func newInterface(nlh *netlink.Handle, config *networkConfiguration) (*bridgeInterface, error) {
	// create interface structure
	i := &bridgeInterface{nlh: nlh}
	// Initialize the bridge name to the default if unspecified.
	if config.BridgeName == "" {
		config.BridgeName = DefaultBridgeName
	}
	// Attempt to find an existing bridge named with the specified name.
	i.Link, err = nlh.LinkByName(config.BridgeName)
	if err != nil {
		logrus.Debugf("Did not find any interface with name %s: %v", config.BridgeName, err)
	}
	//将link转为netlink.Bridge类型 
	else if _, ok := i.Link.(*netlink.Bridge); !ok {
		return nil, fmt.Errorf("existing interface %s is not a bridge", i.Link.Attrs().Name)
	}
	return i, nil
}

func setupDevice(config *networkConfiguration, i *bridgeInterface) error {
	var setMac bool
	// We only attempt to create the bridge when the requested device name is the default one.
	if config.BridgeName != DefaultBridgeName && config.DefaultBridge {
		return NonDefaultBridgeExistError(config.BridgeName)
	}
	// Set the bridgeInterface netlink.Bridge.
	i.Link = &netlink.Bridge{
		LinkAttrs: netlink.LinkAttrs{
			Name: config.BridgeName,
		},
	}
	// Only set the bridge's MAC address if the kernel version is > 3.3, as it was not supported before that.
	kv, err := kernel.GetKernelVersion()
	if err != nil {
		logrus.Errorf("Failed to check kernel versions: %v. Will not assign a MAC address to the bridge interface", err)
	} else {
		setMac = kv.Kernel > 3 || (kv.Kernel == 3 && kv.Major >= 3)
	}
	/*
	LinkAdd adds a new link device. The type and features of the device are taken fromt the parameters in the link object.
	Equivalent to: `ip link add $link`	
	*/
	i.nlh.LinkAdd(i.Link)
	if setMac {
		//随机生成一个mac地址
		hwAddr := netutils.GenerateRandomMAC()
		if err = i.nlh.LinkSetHardwareAddr(i.Link, hwAddr); err != nil {
			return fmt.Errorf("failed to set bridge mac-address %s : %s", hwAddr, err.Error())
		}
		logrus.Debugf("Setting bridge mac address to %s", hwAddr)
	}
	return err
}

func initBridgeDriver(controller libnetwork.NetworkController, config *config.Config) error {
	bridgeName := bridge.DefaultBridgeName
	if config.BridgeConfig.Iface != "" {
		bridgeName = config.BridgeConfig.Iface
	}
	netOption := map[string]string{
		bridge.BridgeName:         bridgeName,
		bridge.DefaultBridge:      strconv.FormatBool(true),
		netlabel.DriverMTU:        strconv.Itoa(config.Mtu),
		bridge.EnableIPMasquerade: strconv.FormatBool(config.BridgeConfig.EnableIPMasq),
		bridge.EnableICC:          strconv.FormatBool(config.BridgeConfig.InterContainerCommunication),
	}
	// --ip processing
	if config.BridgeConfig.DefaultIP != nil {
		netOption[bridge.DefaultBindingIP] = config.BridgeConfig.DefaultIP.String()
	}
	var (
		ipamV4Conf *libnetwork.IpamConf
		ipamV6Conf *libnetwork.IpamConf
	)
	ipamV4Conf = &libnetwork.IpamConf{AuxAddresses: make(map[string]string)}
	/*
	It looks for an interface on the OS with the specified name and returns all its IPv4 and IPv6 addresses in CIDR notation.
	If the interface does not exist, it chooses from a predefined list the first IPv4 address which does not conflict with other interfaces on the system.
	*/
	nwList, nw6List, err := netutils.ElectInterfaceAddresses(bridgeName)
	nw := nwList[0]
	if len(nwList) > 1 && config.BridgeConfig.FixedCIDR != "" {
		//将字符串s解析成一个ip地址和子网掩码的结构体中，*net.IPNet
		_, fCIDR, err := net.ParseCIDR(config.BridgeConfig.FixedCIDR)
	}
	// Iterate through in case there are multiple addresses for the bridge
	for _, entry := range nwList {
		if fCIDR.Contains(entry.IP) {
			nw = entry
			break
		}
	}
	//returns the canonical form for the passed network
	ipamV4Conf.PreferredPool = lntypes.GetIPNetCanonical(nw).String()
	//returns the host portion of the ip address identified by the mask
	//返回主机部分网址
	hip, _ := lntypes.GetHostPartIP(nw.IP, nw.Mask)
	//如果ip是全局单播地址，则返回真。
	if hip.IsGlobalUnicast() {
		ipamV4Conf.Gateway = nw.IP.String()
	}
	if config.BridgeConfig.IP != "" {
		ipamV4Conf.PreferredPool = config.BridgeConfig.IP
		ip, _, err := net.ParseCIDR(config.BridgeConfig.IP)
		ipamV4Conf.Gateway = ip.String()
	} else if bridgeName == bridge.DefaultBridgeName && ipamV4Conf.PreferredPool != "" {
		logrus.Infof("Default bridge (%s) is assigned with an IP address %s. Daemon option --bip can be used to set a preferred IP address", bridgeName, ipamV4Conf.PreferredPool)
	}
	if config.BridgeConfig.FixedCIDR != "" {
		_, fCIDR, err := net.ParseCIDR(config.BridgeConfig.FixedCIDR)
		ipamV4Conf.SubPool = fCIDR.String()
	}
	if config.BridgeConfig.DefaultGatewayIPv4 != nil {
		ipamV4Conf.AuxAddresses["DefaultGatewayIPv4"] = config.BridgeConfig.DefaultGatewayIPv4.String()
	}
	//ipv6地址的设置
	var deferIPv6Alloc bool
	if config.BridgeConfig.FixedCIDRv6 != "" {
		_, fCIDRv6, err := net.ParseCIDR(config.BridgeConfig.FixedCIDRv6)
		ones, _ := fCIDRv6.Mask.Size()
		deferIPv6Alloc = ones <= 80
		if ipamV6Conf == nil {
			ipamV6Conf = &libnetwork.IpamConf{AuxAddresses: make(map[string]string)}
		}
		ipamV6Conf.PreferredPool = fCIDRv6.String()
		for _, nw6 := range nw6List {
			if fCIDRv6.Contains(nw6.IP) {
				ipamV6Conf.Gateway = nw6.IP.String()
				break
			}
		}
	}
	if config.BridgeConfig.DefaultGatewayIPv6 != nil {
		if ipamV6Conf == nil {
			ipamV6Conf = &libnetwork.IpamConf{AuxAddresses: make(map[string]string)}
		}
		ipamV6Conf.AuxAddresses["DefaultGatewayIPv6"] = config.BridgeConfig.DefaultGatewayIPv6.String()
	}
	v4Conf := []*libnetwork.IpamConf{ipamV4Conf}
	v6Conf := []*libnetwork.IpamConf{}
	if ipamV6Conf != nil {
		v6Conf = append(v6Conf, ipamV6Conf)
	}
	// Initialize default network on "bridge" with the same name,这里上文已经分析过
	_, err = controller.NewNetwork("bridge", "bridge", "",
		libnetwork.NetworkOptionEnableIPv6(config.BridgeConfig.EnableIPv6),
		libnetwork.NetworkOptionDriverOpts(netOption),
		libnetwork.NetworkOptionIpam("default", "", v4Conf, v6Conf, nil),
		libnetwork.NetworkOptionDeferIPv6Alloc(deferIPv6Alloc))
	return nil
}

func (daemon *Daemon) allocateNetwork(container *container.Container) error {
	//取得daemon中的netController
	controller := daemon.netController
	
	/*
		always connect default network first since only default network mode support link and we need do some setting on sandbox initialize for link, but the sandbox only be initialized on first network connecting.
		总是先连接default network，因为只有default network支持link,而我们需要在sandbox初始化时为link做一些设置，而sandbox只在连接第一个网络时初始化
	*/
	//返回daemon使用的默认网络栈
	defaultNetName := runconfig.DefaultDaemonNetworkMode().NetworkName()
	if nConf, ok := container.NetworkSettings.Networks[defaultNetName]; ok {
		//重置操作数据的endpoint设置
		cleanOperationalData(nConf)
		//将sandbox连接到defaultNetwork
		if err := daemon.connectToNetwork(container, defaultNetName, nConf.EndpointSettings, updateSettings); err != nil {
			return err
		}

	}
	
	//将container.NetworkSettings.Networks中的数据取出，存入networks中，这样做是防止connectToNetwork()函数修改container.NetworkSettings.Networks的值
	networks := make(map[string]*network.EndpointSettings)
	for n, epConf := range container.NetworkSettings.Networks {
		if n == defaultNetName {
			continue
		}

		networks[n] = epConf
	}
	
	//遍历container.NetworkSettings.Networks的网络，依次将这些网络加入到container中
	for netName, epConf := range networks {
		cleanOperationalData(epConf)
		if err := daemon.connectToNetwork(container, netName, epConf.EndpointSettings, updateSettings); err != nil {
			return err
		}
	}

	//如果container没有连接到任何网络
	//现在就建立它的sandbox，对于别的情况应该是在connectToNetwork()函数中创建的
	if len(networks) == 0 {
		//如果container还没有sandbox
		if nil == daemon.getNetworkSandbox(container) {
			//为sandbox创建配置信息
			options, err := daemon.buildSandboxOptions(container)
			//为container新建一个sandbox
			sb, err := daemon.netController.NewSandbox(container.ID, options...)
			//更新sandbox的ID和key
			container.UpdateSandboxNetworkSettings(sb)
		}
	}

	//saves the host configuration on disk for the container.
	if err := container.WriteHostConfig(); err != nil {
		return err
	}
	networkActions.WithValues("allocate").UpdateSince(start)
	return nil
}

//以daemon.connectToNetwork(container, netName, epConf.EndpointSettings, updateSettings)为例
func (daemon *Daemon) connectToNetwork(container *container.Container, idOrName string, endpointConfig *networktypes.EndpointSettings, updateSettings bool) (err error) {
	//如果是container模式，返回错误
	if container.HostConfig.NetworkMode.IsContainer() {
		return runconfig.ErrConflictSharedNetwork
	}
	//根据network的name或id查找network，并连接，在3.3.1中再详细分析
	n, config, err := daemon.findAndAttachNetwork(container, idOrName, endpointConfig)
	
	var operIPAM bool
	if config != nil {
		if epConfig, ok := config.EndpointsConfig[n.Name()]; ok {
			if endpointConfig.IPAMConfig == nil ||
				(endpointConfig.IPAMConfig.IPv4Address == "" &&
					endpointConfig.IPAMConfig.IPv6Address == "" &&
					len(endpointConfig.IPAMConfig.LinkLocalIPs) == 0) {
				operIPAM = true
			}
			// 将连接的network的epConfig的IPAMConcif和NetworkID赋值到endpointConfig中，用于之后的网络配置更新
			endpointConfig.IPAMConfig = epConfig.IPAMConfig
			endpointConfig.NetworkID = epConfig.NetworkID
		}
	}
	//更新容器的网络配置
	err = daemon.updateNetworkConfig(container, n, endpointConfig, updateSettings)
	//获得netController
	controller := daemon.netController
	//获得容器的sandbox,从controller的Sandbox数组中获取,在3.3.2中分析
	sb := daemon.getNetworkSandbox(container)
	//创建endpoint的配置信息
	createOptions, err := container.BuildCreateEndpointOptions(n, endpointConfig, sb, daemon.configStore.DNS)
	//endPoint的名字
	endpointName := strings.TrimPrefix(container.Name, "/")
	//创建endpoint,下面在3.3.3中分析
	ep, err := n.CreateEndpoint(endpointName, createOptions...)
	//将network的endpoint信息赋值到NetworkSettings中
	container.NetworkSettings.Networks[n.Name()] = &network.EndpointSettings{
		EndpointSettings: endpointConfig,
		IPAMOperational:  operIPAM,
	}
	
	if _, ok := container.NetworkSettings.Networks[n.ID()]; ok {
		delete(container.NetworkSettings.Networks, n.ID())
	}
	//更新Endpoint信息
	if err := daemon.updateEndpointNetworkSettings(container, n, ep); err != nil {
		return err
	}
	//如果sandbox为空，则新建一个
	if sb == nil {
		options, err := daemon.buildSandboxOptions(container)
		if err != nil {
			return err
		}
		//在3.3.2中分析
		sb, err = controller.NewSandbox(container.ID, options...)
		if err != nil {
			return err
		}

		container.UpdateSandboxNetworkSettings(sb)
	}
	//根据network创建endpoint join的配置信息
	joinOptions, err := container.BuildJoinOptions(n)
	//将endpoint加入sandbox中，将在3.3.4分析
	ep.Join(sb, joinOptions...)
	//start中传入的managed为false
	if !container.Managed {
		// add container name/alias to DNS
		if err := daemon.ActivateContainerServiceBinding(container.Name); err != nil {
			return fmt.Errorf("Activate container service binding for %s failed: %v", container.Name, err)
		}
	}
	//当container加入endpoint时，更新network的配置
	if err := container.UpdateJoinInfo(n, ep); err != nil {
		return fmt.Errorf("Updating join info failed: %v", err)
	}
	//这里调用的是container.GetSandboxPortMapInfo，返回sandbox的端口映射情况
	container.NetworkSettings.Ports = getPortMapInfo(sb)

	daemon.LogNetworkEventWithAttributes(n, "connect", map[string]string{"container": container.ID})
	networkActions.WithValues("connect").UpdateSince(start)
	return nil
}

func (daemon *Daemon) findAndAttachNetwork(container *container.Container, idOrName string, epConfig *networktypes.EndpointSettings) (libnetwork.Network, *networktypes.NetworkingConfig, error) {
	//根据name或者id查找network
	n, err := daemon.FindNetwork(idOrName)
	// If we found a network and if it is not dynamically created
	// we should never attempt to attach to that network here.
	if n != nil {
		if container.Managed || !n.Info().Dynamic() {
			return n, nil, nil
		}
	}
	//将IPAMConfig中的IPv4和IPv6的地址添加到addresses数组中
	var addresses []string
	if epConfig != nil && epConfig.IPAMConfig != nil {
		if epConfig.IPAMConfig.IPv4Address != "" {
			addresses = append(addresses, epConfig.IPAMConfig.IPv4Address)
		}

		if epConfig.IPAMConfig.IPv6Address != "" {
			addresses = append(addresses, epConfig.IPAMConfig.IPv6Address)
		}
	}
	var (
		config     *networktypes.NetworkingConfig
		retryCount int
	)
	//这段对cluster网络的处理
	for {
		// In all other cases, attempt to attach to the network to
		// trigger attachment in the swarm cluster manager.  
		//其他情况下，尝试连接到swarm cluster manager的网络触发附件中
		//？？？这端不是很懂
		//对cluster网络的处理
		if daemon.clusterProvider != nil {
			var err error
			config, err = daemon.clusterProvider.AttachNetwork(idOrName, container.ID, addresses)
			if err != nil {
				return nil, nil, err
			}
		}
		n, err = daemon.FindNetwork(idOrName)
		if err != nil {
			if daemon.clusterProvider != nil {
				if err := daemon.clusterProvider.DetachNetwork(idOrName, container.ID); err != nil {
					logrus.Warnf("Could not rollback attachment for container %s to network %s: %v", container.ID, idOrName, err)
				}
			}

			// Retry network attach again if we failed to
			// find the network after successful
			// attachment because the only reason that
			// would happen is if some other container
			// attached to the swarm scope network went down
			// and removed the network while we were in
			// the process of attaching.
			if config != nil {
				if _, ok := err.(libnetwork.ErrNoSuchNetwork); ok {
					if retryCount >= 5 {
						return nil, nil, fmt.Errorf("could not find network %s after successful attachment", idOrName)
					}
					retryCount++
					continue
				}
			}

			return nil, nil, err
		}

		break
	}
	// This container has attachment to a swarm scope
	// network. Update the container network settings accordingly.
	container.NetworkSettings.HasSwarmEndpoint = true
	return n, config, nil
}	

func (daemon *Daemon) FindNetwork(idName string) (libnetwork.Network, error) {
	//根据name查找
	n, err := daemon.GetNetworkByName(idName)
	if n != nil {
		return n, nil
	}
	//根据id查找
	return daemon.GetNetworkByID(idName)
}

func (daemon *Daemon) GetNetworkByName(name string) (libnetwork.Network, error) {
	//获得controller
	c := daemon.netController
	if name == "" {
		name = c.Config().Daemon.DefaultNetwork
	}
	return c.NetworkByName(name)
}

func (c *controller) NetworkByName(name string) (Network, error) {
	var n Network
	//查找network时，确认是所查network用到的函数
	s := func(current Network) bool {
		if current.Name() == name {
			n = current
			return true
		}
		return false
	}
	//遍历controller中的network
	c.WalkNetworks(s)
	return n, nil
}

func (c *controller) WalkNetworks(walker NetworkWalker) {
	for _, n := range c.Networks() {
		if walker(n) {
			return
		}
	}
}

func (c *controller) Networks() []Network {
	var list []Network
	networks, err := c.getNetworksFromStore()
	for _, n := range networks {
		if n.inDelete {
			continue
		}
		list = append(list, n)
	}
	return list
}

func (c *controller) getNetworksFromStore() ([]*network, error) {
	var nl []*network
	//遍历controller的stores,类型为[]datastore.DataStore
	for _, store := range c.getStores() {
		kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix),
			&network{ctrlr: c})
		// Continue searching in the next store if no keys found in this store
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				logrus.Debugf("failed to get networks for scope %s: %v", store.Scope(), err)
			}
			continue
		}

		kvep, err := store.Map(datastore.Key(epCntKeyPrefix), &endpointCnt{})
		if err != nil {
			if err != datastore.ErrKeyNotFound {
				logrus.Warnf("failed to get endpoint_count map for scope %s: %v", store.Scope(), err)
			}
		}

		for _, kvo := range kvol {
			n := kvo.(*network)
			n.Lock()
			n.ctrlr = c
			ec := &endpointCnt{n: n}
			// Trim the leading & trailing "/" to make it consistent across all stores
			if val, ok := kvep[strings.Trim(datastore.Key(ec.Key()...), "/")]; ok {
				ec = val.(*endpointCnt)
				ec.n = n
				n.epCnt = ec
			}
			n.scope = store.Scope()
			n.Unlock()
			nl = append(nl, n)
		}
	}
	return nl, nil
}

func (daemon *Daemon) getNetworkSandbox(container *container.Container) libnetwork.Sandbox {
	var sb libnetwork.Sandbox
	daemon.netController.WalkSandboxes(func(s libnetwork.Sandbox) bool {
		if s.ContainerID() == container.ID {
			sb = s
			return true
		}
		return false
	})
	return sb
}

func (c *controller) WalkSandboxes(walker SandboxWalker) {
	for _, sb := range c.Sandboxes() {
		if walker(sb) {
			return
		}
	}
}

func (c *controller) Sandboxes() []Sandbox {
	list := make([]Sandbox, 0, len(c.sandboxes))
	//sandboxes为sandboxTable类型
	for _, s := range c.sandboxes {
		// Hide stub sandboxes from libnetwork users
		if s.isStub {
			continue
		}
		list = append(list, s)
	}
	return list
}

func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (sBox Sandbox, err error) {
	var sb *sandbox
	c.Lock()
	//对于container已经有sandbox的处理
	for _, s := range c.sandboxes {
		if s.containerID == containerID {
			// If not a stub, then we already have a complete sandbox.
			if !s.isStub {
				sbID := s.ID()
				c.Unlock()
				return nil, types.ForbiddenErrorf("container %s is already present in sandbox %s", containerID, sbID)
			}

			// We already have a stub sandbox from the
			// store. Make use of it so that we don't lose
			// the endpoints from store but reset the
			// isStub flag.
			sb = s
			sb.isStub = false
			break
		}
	}
	c.Unlock()
	//首先创建sandbox和process options,Key的生成由option决定
	if sb == nil {
		sb = &sandbox{
			id:                 stringid.GenerateRandomID(),
			containerID:        containerID,
			endpoints:          epHeap{},
			epPriority:         map[string]int{},
			populatedEndpoints: map[string]struct{}{},
			config:             containerConfig{},
			controller:         c,
			extDNS:             []extDNSEntry{},
		}
	}
	sBox = sb
	//调用的Golang的"container/heap"包，将sandbox的endpoints初始化为heap操作
	heap.Init(&sb.endpoints)
	//处理options
	sb.processOptions(options...)
	c.Lock()
	if sb.ingress && c.ingressSandbox != nil {
		c.Unlock()
		return nil, types.ForbiddenErrorf("ingress sandbox already present")
	}
	//将新建的sandbox赋值给controller的ingressSandbox
	if sb.ingress {
		c.ingressSandbox = sb
		sb.config.hostsPath = filepath.Join(c.cfg.Daemon.DataDir, "/network/files/hosts")
		sb.config.resolvConfPath = filepath.Join(c.cfg.Daemon.DataDir, "/network/files/resolv.conf")
		sb.id = "ingress_sbox"
	}
	c.Unlock()
	//创建host file、更新parentHosts、启动DNS
	if err = sb.setupResolutionFiles(); err != nil {
		return nil, err
	}
	//使用默认sandbox
	if sb.config.useDefaultSandBox {
		c.sboxOnce.Do(func() {
			/*
				NewSandbox provides a new sandbox instance created in an os specific way  provided a key which uniquely identifies the sandbox
				下面继续分析
			*/
			c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false)
		})
		//osSbox类型为osl.Sandbox，Package osl describes structures and interfaces which abstract os entities
		sb.osSbox = c.defOsSbox
	}
	//将新建的sandbox注册到controller的sandboxes数组中
	c.Lock()
	c.sandboxes[sb.id] = sb
	c.Unlock()
	err = sb.storeUpdate()
	return sb, nil
}

func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
	//根据上文可以知道，这里isRestore是false，所以会执行下面的代码
	if !isRestore {
		//创建network namespace，下面接着分析
		err := createNetworkNamespace(key, osCreate)
		if err != nil {
			return nil, err
		}
	} else {
		//createBasePath()函数只执行一次
		once.Do(createBasePath)
	}
	n := &networkNamespace{path: key, isDefault: !osCreate, nextIfIndex: make(map[string]int)}
	//get network namespace
	sboxNs, err := netns.GetFromPath(n.path)
	//create a netlink handle
	n.nlHandle, err = netlink.NewHandleAt(sboxNs, syscall.NETLINK_ROUTE)
	//set the timeout on the sandbox netlink handle sockets
	n.nlHandle.SetSocketTimeout(ns.NetlinkSocketsTimeout)
	
	// As starting point, disable IPv6 on all interfaces
	if !n.isDefault {
		//disable IPv6 on all interfaces on network namespace
		setIPv6(n.path, "all", false)
	}
	//set up lookback
	n.loopbackUp()
	//返回该namespace
	return n, nil
}

func createNetworkNamespace(path string, osCreate bool) error {
	//创建namespace文件
	createNamespaceFile(path)
	cmd := &exec.Cmd{
		//Returns "/proc/self/exe"，代表当前程序
		Path:   reexec.Self(),
		Args:   append([]string{"netns-create"}, path),
		Stdout: os.Stdout,
		Stderr: os.Stderr,
	}
	//从上文可以看到，osCreate为false，所以下面这段代码不执行
	if osCreate {
		cmd.SysProcAttr = &syscall.SysProcAttr{}
		cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNET
	}
	//namespace creation reexec command
	cmd.Run()
	return nil
}