U Ad+@sddlZddlZddlZddlZddlZddlmZddlmZm Z ddl m Z m Z m Z mZmZddlmZmZddlmZddlmZmZddlmZdd lmZdd lmZdd lmZdd lmZm Z dd l!m"Z"ddl#m$Z$ddl%m&Z&ddl'm(Z(dZ)dZ*dZ+dZ,e)e*e+gZ-dZ.dZ/dZ0dZ1dZ2dZ3dZ4dZ5e6e7Z8dddfd d!dfd"d#dfd$Z9e Gd%d&d&eZ:Gd'd(d(e;ZdWd/d0Z?e2fd1d2Z@ed3d4d5d6d7gZAed8d9d:gZBGd;dde&ejCd<ZDd=d>ZEeeDeFfd?d@dAZGdBdCZHdXeFeIdEdFdGZJdHdIZKdYdKdLZLGdMdNdNeMZNdOdPZOeDeFeIdQdRdSZPeFeeDdTdUdVZQdS)ZN) namedtuple)Enumunique)AnyDictListOptionalTuple)dmiimporter)log)net type_utils) user_data)util) write_json)Distro) EventScope EventType) launch_index)Paths)CloudInitPickleMixin)eventsZdisabledlocalr passZ FILESYSTEMNETWORK DataSourcez|EXPERIMENTAL: The structure and format of content scoped under the 'ds' key may change in subsequent releases of cloud-init.zredacted for non-root user cloud-nameZ_unsetunknownz aws-chinacCs|dkSNZawscr r ?r$zaws-govcCs|dkSrr r!r r r#r$@r%z azure-chinacCs|dkS)NZazurer r!r r r#r$Ar%)zcn-zus-gov-Zchinac@s2eZdZdZdZdZdZdZdZe ddd Z d S) NetworkConfigSourcezb Represents the canonical list of network config sources that cloud-init knows about. ZcmdlinedsZ system_cfgZfallbackZ initramfsreturncCs|jSNvalueselfr r r#__str__RszNetworkConfigSource.__str__N) __name__ __module__ __qualname____doc__CMD_LINEDS SYSTEM_CFGZFALLBACK INITRAMFSstrr/r r r r#r&Esr&c@seZdZdZdS)DatasourceUnpickleUserDataErrorzERaised when userdata is unable to be unpickled due to python upgradesNr0r1r2r3r r r r#r9Vsr9c@s eZdZdS)DataSourceNotFoundExceptionNr0r1r2r r r r#r;Zsr;c@seZdZdZdS)InvalidMetaDataExceptionz8Raised when metadata is broken, unavailable or disabled.Nr:r r r r#r=^sr=r c Cst|}g}g}|D]\}}|r4|d|}n|}||ksP||krZ||t|tr|dr|||dd||<t|t rt |||} | | d| | d| ||<qt ||d<t ||d<|S)zProcess all instance metadata cleaning it up for persisting as json. Strip ci-b64 prefix and catalog any 'base64_encoded_keys' as a list @return Dict copy of processed metadata. /zci-b64:r>base64_encoded_keyssensitive_keys)copydeepcopyitemslowerappend isinstancer8 startswithreplacedictprocess_instance_metadataextendpopsorted) metadatakey_pathrAmd_copyr@Z sens_keyskeyvalZ sub_key_pathZ return_valr r r#rKbs6         rKcCs|dgs|St|}|dD]V}|d}|}|D].}||kr:t||tr:||dkr:||}q:||kr$|||<q$|S)zRedact any sensitive keys from to provided metadata dictionary. Replace any keys values listed in 'sensitive_keys' with redact_value. rAr?)getrBrCsplitrGrJ)rOZ redact_valuerQrP path_partsobjpathr r r#redact_sensitive_keyss"       rZ URLParamsZmax_wait_secondsZtimeout_secondsZ num_retriessec_between_retriesDataSourceHostnamehostname is_defaultc @seZdZUeZdZdZdZdZdZ dZ e j e j e je jfZee dfed<dZdZdZd ZejejejejejhiZejejhiZd e fd e fd ifd dddddf Z!eee"e#fdfed<dZ$dZ%ee"dfed<dZ&d Z'dze(e)dddZ*e+ddddZ,ddZ-e.dd d!Z/d"d#Z0d$d%Z1d&d'Z2d{d)d*Z3e.dd+d,Z4d|d.d/Z5e.dd0d1Z6d2d3Z7d}d4d5Z8d6d7Z9d8d9Z:e;d:d;Zd?Z>d@dAZ?e;dBdCZ@dDdEZAe;dFdGZBdHdIZCe;dJdKZDdLdMZEdNdOZFdPdQZGdRdSZHdTdUZIdVdWZJdXdYZKdZd[ZLd\d]ZMe;d^d_ZNe;d`daZOdbdcZPd~dddeZQdfdgZReSedhdidjZTeSee.dkdldmZUdndoZVeWddpdqZXe;drdsZYe;dtduZZdvdwZ[dxdyZ\dS)rz en_US.UTF-8Z_undefN.network_config_sourcesrT  ec2_metadata network_jsonrO)userdataN) userdata_rawN) vendordataN)vendordata_rawN) vendordata2N)vendordata2_rawNcached_attr_defaultsF) merged_cfgzsecurity-credentialsrf user-datarrh vendor-datazds/vendor_datasensitive_metadata_keys)distropathscCs~||_||_||_d|_i|_d|_d|_d|_d|_d|_ t |jd|j fi|_ |j s`i|_ |stt|j|_n||_dS)NZ datasource)sys_cfgrqrrrfrOrgrhrjrirkrZget_cfg_by_pathdsnameds_cfgudZUserDataProcessorud_proc)r.rsrqrrrwr r r#__init__s(zDataSource.__init__)ci_pkl_versionr)c Cst|dsd|_t|ds d|_t|ds0d|_t|dr|jdk rzt|jWn6tk r}ztd|t |W5d}~XYnXdS)z(Perform deserialization fixes for Paths.rjNrkskip_hotplug_detectFrfz:Unable to unpickle datasource: %s. Ignoring current cache.) hasattrrjrkrzrfr8AttributeErrorLOGdebugr9)r.ryer r r# _unpickle1s   zDataSource._unpicklecCs t|Sr*robj_namer-r r r#r/IszDataSource.__str__r(cCsdS)z#Check if running on this datasourceTr r-r r r# ds_detectLszDataSource.ds_detectcCs |jdg|jg|jdgfkS)zOverride if either: - only a single datasource defined (nothing to fall back to) - TODO: commandline argument is used (ci.ds=OpenStack) Zdatasource_listNone)rsrUrtr-r r r#override_ds_detectPs zDataSource.override_ds_detectcCsD|rtd|n&|r,td|ntd|dS|S)z&Overrides runtime datasource detectionz5Machine is configured to run on single datasource %s.zMachine is running on %s.z#Datasource type %s is not detected.F)rr}r~r _get_datar-r r r#_check_and_get_dataZs zDataSource._check_and_get_datacCs|j}|}|j}|d}ddg||t|j|j|j|j|j|dd|dd|dd|j||d|||d d|||d d |j|j |d |d d iS)z2Return a dictionary of standardized metadata keys.sys_infov1 subplatformZdistrrcpythonunameplatformvariant)Z _beta_keysavailability-zoneavailability_zonecloud_idr cloud_namerqZdistro_versionZdistro_releaserZpublic_ssh_keysZpython_version instance-id instance_idZkernel_releaselocal-hostnamelocal_hostnamemachineregionrZsystem_platformr) get_hostnamer^get_instance_idrcanonical_cloud_idrr platform_typeget_public_ssh_keysr)r. instance_datarrrZsysinfor r r#_get_standardized_metadatagsB      z%DataSource._get_standardized_metadatar cCsL|js dS|r|}n|j}|D]\}}t||rt|||q|sHd|_dS)zReset any cached metadata attributes to datasource defaults. @param attr_defaults: Optional tuple of (attr, value) pairs to set instead of cached_attr_defaults. NF) _dirty_cacherlr{setattr)r.Z attr_defaultsZ attr_valuesZ attributer,r r r#clear_cached_attrss  zDataSource.clear_cached_attrscCs"d|_|}|s|S||S)zDatasources implement _get_data to setup metadata and userdata_raw. Minimally, the datasource should return a boolean True on success. T)rrpersist_instance_data)r.Z return_valuer r r#get_datas zDataSource.get_dataTc Csf|r&tj|jjr&t||jdt|drbt t |d}| dd| ddd|i}n^dd|j ii}t|drt |d}|t kr||dd<t|d rt |d }|t kr||dd <t|dd <t |j|d <d |d d <t|d <|||z"t|}tt||jd}Wnrtk r`}ztdt|WYdSd}~XYn:tk r}ztdt|WYdSd}~XYnX|jd} |ddd} tj |jj!d} t"| d| | dd} | d| } tj#| rtj$| } tj%| | dd| r:| | kr:t&| t'| |dd|jd}t'|t(|dS)aPProcess and write INSTANCE_JSON_FILE with all instance metadata. Replace any hyphens with underscores in key names for use in template processing. :param write_cache: boolean set True to persist obj.pkl when instance_link exists. @return True on successful write, False otherwise. Zobj_pklZ_crawled_metadatarnNror'Z meta_datarerdZ_docrmzUMerged cloud-init system config from /etc/cloud/cloud.cfg and /etc/cloud/cloud.cfg.d/r)rAz'Error persisting instance-data.json: %sFZinstance_data_sensitiverrZnonezcloud-id- T)Zforcei)moder))osrYlexistsrrZ instance_link pkl_storeZ get_ipath_curr{rBrCgetattrrMrOUNSETEXPERIMENTAL_TEXTrsrZ system_infoupdaterZ json_dumpsrKjsonloadsrp TypeErrorr}warningr8UnicodeDecodeErrorZ get_runpathrUjoinZrun_dir write_fileexistsrealpathZsym_linkZdel_filerrZ)r.Z write_cacheZcrawled_metadatarrerdZcontentZprocessed_datarZjson_sensitive_filerZ cloud_id_fileZprev_cloud_id_fileZnew_cloud_id_fileZ json_filer r r#rsf                    z DataSource.persist_instance_datacCs tddS)z@Walk metadata sources, process crawled data and save attributes.zlSubclasses of DataSource must implement _get_data which sets self.metadata, vendordata_raw and userdata_raw.N)NotImplementedErrorr-r r r#rszDataSource._get_datac CsL|j}zt|jd|j}Wn,tk rJttd|jd|YnX|j}zt dt|jd|j}Wn2tk r|j}ttd|jd|YnX|j }zt|jd|j }Wn,t k rttd|jd|YnX|j }zt|jd|j }Wn.t k r<ttd |jd|YnXt ||||S) zReturn the Datasource's preferred url_read parameters. Subclasses may override url_max_wait, url_timeout, url_retries. @return: A URLParams object with max_wait_seconds, timeout_seconds, num_retries. max_waitz6Config max_wait '%s' is not an int, using default '%s'rtimeoutz5Config timeout '%s' is not an int, using default '%s'retriesz5Config retries '%s' is not an int, using default '%s'r\zAConfig sec_between_retries '%s' is not an int, using default '%s') url_max_waitintrurU ValueErrorrlogexcr} url_timeoutmax url_retries Exceptionurl_sec_between_retriesr[)r.rrrr\r r r#get_url_paramss^        zDataSource.get_url_paramscCs2|jdkr|j||_|r,||jS|jSr*)rfrwprocessget_userdata_raw _filter_xdata)r.Z apply_filterr r r# get_userdata:s   zDataSource.get_userdatacCs"|jdkr|j||_|jSr*)rhrwrget_vendordata_rawr-r r r#get_vendordataAs zDataSource.get_vendordatacCs"|jdkr|j||_|jSr*)rjrwrget_vendordata2_rawr-r r r#get_vendordata2Fs zDataSource.get_vendordata2cCs2|jdkr,t|_|jdkr,td|j|jS)zADetermine the network interface used during local network config.Nz(Did not find a fallback interface on %s.)_fallback_interfacer Zfind_fallback_nicr}rrr-r r r#fallback_interfaceKs   zDataSource.fallback_interfacecCs.t|ds|j|_|js(|j|_|jS)N_platform_type)r{rtrErr-r r r#rVs    zDataSource.platform_typecCs*t|ds||_|js$||_|jS)aReturn a string representing subplatform details for the datasource. This should be guidance for where the metadata is sourced. Examples of this on different clouds: ec2: metadata (http://169.254.169.254) openstack: configdrive (/dev/path) openstack: metadata (http://169.254.169.254) nocloud: seed-dir (/seed/dir/path) lxd: nocloud (/seed/dir/path) _subplatform)r{_get_subplatformrr-r r r#r_s   zDataSource.subplatformcCst|drdt|dStS)z?Subclasses should implement to return a "slug (detail)" string.Zmetadata_addressz metadata (%s))r{rMETADATA_UNKNOWNr-r r r#rrs zDataSource._get_subplatformcCsv|jr |jS|jrb|jtrb|jt}t|tr@||_qp||_t dtt |n||_|jS)zReturn lowercase cloud name as determined by the datasource. Datasource can determine or define its own cloud product name in metadata. z5Ignoring metadata provided key %s: non-string type %s) _cloud_namerOrUMETADATA_CLOUD_NAME_KEYrGr8rE_get_cloud_namer}r~type)r.rr r r#rxs   zDataSource.cloud_namecCs|jS)zReturn the datasource name as it frequently matches cloud name. Should be overridden in subclasses which can run on multiple cloud names, such as DatasourceEc2. )rtr-r r r#rszDataSource._get_cloud_namecCs"|js dSd|jkr|jdSdS)Nz launch-index)rOr-r r r#rs   zDataSource.launch_indexcCs0tt|jg}|}|D]}||}q|Sr*)rZFilterrZsafe_intZapply)r.Z processed_udfiltersZnew_udfr r r#rs  zDataSource._filter_xdatacCsdSNFr r-r r r#is_disconnectedszDataSource.is_disconnectedcCs|jSr*)rgr-r r r#rszDataSource.get_userdata_rawcCs|jSr*)rir-r r r#rszDataSource.get_vendordata_rawcCs|jSr*)rkr-r r r#rszDataSource.get_vendordata2_rawcCsiSr*r r-r r r#get_config_objszDataSource.get_config_objcCst|jdS)Nz public-keys)normalize_pubkey_datarOrUr-r r r#rszDataSource.get_public_ssh_keyscCsdS)a5Publish the public SSH host keys (found in /etc/ssh/*.pub). @param hostkeys: List of host key tuples (key_type, key_value), where key_type is the first field in the public key file (e.g. 'ssh-rsa') and key_value is the key itself (e.g. 'AAAAB3NzaC1y...'). Nr )r.Zhostkeysr r r#publish_host_keysszDataSource.publish_host_keyscCsdddi}|D]N\}}||s$q|D]4}d||t|df}tj|r(|Sq(qdS)NZsd)ZvdZxvdZvtbz /dev/%s%s)rDrHlenrrYr)r.Z short_nameZmappingsZnfromZtlistZntoZcandr r r# _remap_devices  zDataSource._remap_devicecCsdSr*r )r._namer r r#device_name_to_devicesz DataSource.device_name_to_devicecCs.|j}z|j}Wntk r(YnX|S)z, (str, bool). is_default is a bool and it's true only if hostname is localhost and was returned by util.get_hostname() as a default. This is used to differentiate with a user-defined localhost hostname. Optionally return (None, False) when metadata_only is True and local-hostname data is not available. Z localdomainZ localhostFrNT.rzip-%srrcz%s.%s)rOrUr]rrZget_fqdn_from_hostsfindr8rVr Zis_ipv4_addressZ gethostbyaddrrIrr) r.ZfqdnZ resolve_ipZ metadata_onlyZ defdomainZdefhostZdomainr_Ztoksr^Z hosts_fqdnZlhostr r r#rsF         zDataSource.get_hostnamecCs|jj|dS)N)Z data_source)rqget_package_mirror_infor-r r r#rDsz"DataSource.get_package_mirror_info)source_event_typescCsPi}|D]B}|jD]2\}}||kr||s:t||<|||qq|Sr*)supported_update_eventsrDrUsetadd)r.rsupported_eventseventZ update_scopeZ update_eventsr r r#get_supported_eventsGs  zDataSource.get_supported_events)rr)c Cs||}|D]<\}}td|jddd|D|d|tffq|rl||}|rldStd|ddd|Dd S) aRefresh cached metadata if the datasource supports this event. The datasource has a list of supported_update_events which trigger refreshing all cached metadata as well as refreshing the network configuration. @param source_event_types: List of EventTypes which may trigger a metadata update. @return True if the datasource did successfully update cached metadata due to source_event_type. z:Update datasource metadata and %s config due to events: %s, cSsg|] }|jqSr r+.0rr r r# hsz;DataSource.update_metadata_if_supported..z _%s_configTz(Datasource %s not updated for events: %scSsg|] }|jqSr r+rr r r#rvsF) rrDr}r~r,rrrr)r.rrZscopeZmatched_eventsresultr r r#update_metadata_if_supportedTs& z'DataSource.update_metadata_if_supportedcCsdSrr )r.rsr r r#check_instance_idzszDataSource.check_instance_idcCsV|dkr t}|dkrt}|D]4}|dkr*q||kr:|Std|||Sq|S)Nz%invalid dsmode '%s', using default=%s)DSMODE_NETWORK VALID_DSMODESr}r)Z candidatesdefaultZvalid candidater r r#_determine_dsmode~s  zDataSource._determine_dsmodecCsdSr*r r-r r r#network_configszDataSource.network_configcCsdSr*r r-r r r#first_instance_bootszDataSource.first_instance_bootcCsdS)a(setup(is_new_instance) This is called before user-data and vendor-data have been processed. Unless the datasource has set mode to 'local', then networking per 'fallback' or per 'network_config' will have been written and brought up the OS at this point. Nr )r.is_new_instancer r r#setups zDataSource.setupcCsdS)aactivate(cfg, is_new_instance) This is called before the init_modules will be called but after the user-data and vendor-data have been fully processed. The cfg is fully up to date config, it contains a merged view of system config, datasource config, user config, vendor config. It should be used rather than the sys_cfg passed to __init__. is_new_instance is a boolean indicating if this is a new instance. Nr )r.Zcfgrr r r#activates zDataSource.activate)N)r )T)F)FFF)NN)]r0r1r2rZdsmoderrtrrrrr&r4r7r6r5r`r __annotations__rrrrrrrBOOT_NEW_INSTANCEZBOOTZ BOOT_LEGACYZHOTPLUGrZdefault_update_eventsrrlr8rrrprzZ_ci_pkl_versionrrrxrrr/boolrrrrrrrrrrrrpropertyrrrrrrrrrrrrrrrrrrrrrrrrrrr staticmethodrrrrrr r r r#rs      &  K<             H &    ) metaclasscCsg}|s |St|tr|St|ttfr4t|St|tr|D]>\}}t|tr^|g}t|ttfrF|D]}|rp||qpqF|Sr*)rGr8 splitlineslistrrJrDrF)Z pubkey_datakeysZ_keynameZklistZpkeyr r r#rs    rr(c Cs"t|||}dd|D}t|kr&dnd} td| |t||D]\} } tjd| ddd | | fd | | f|d } zh| Ztd | | |||} | t j grd | | f| _ | t | fW5QRWSW5QRXWqBtk rttd| YqBXqBdd|}t|dS)NcSsg|]}t|qSr r)rrr r r#rszfind_source..Znetworkrz#Searching for %s data source in: %sz search-%srr>zsearching for %s data from %szno %s data found from %s)nameZ descriptionmessageparentz%Seeing if we can get any data from %szfound %s data from %szGetting data from %s failedz4Did not find any data source, searched classes: (%s)r) list_sources DEP_NETWORKr}r~ziprZReportEventStackrIrrrr rrrrrrr;)rsrqrrZds_depscfg_listpkg_listZreporterds_listZds_namesrr clsZmyrepsmsgr r r# find_sources4     .rc Csg}td||||D]v}|ts2dt|f}t||dg\}}|sVtd||D]2}t|}t|d} | |} | rZ| | qqZq|S)NzLLooking for data source in: %s, via packages %s that matches dependencies %sz%s%sZget_datasource_listzDCould not import %s. Does the DataSource exist and is it importable?) r}r~rH DS_PREFIXr find_moduleerror import_modulerrL) rdependsrZsrc_listZds_nameZm_locsZ _looked_locsZm_locmodZlisterZmatchesr r r#rs8     r system-uuid)fieldr)cCs*|sdSt|}|sdS||kSr)r Z read_dmi_datarE)rr Z dmi_valuer r r#instance_id_matches_system_uuids  r!cCsl|st}|st}|tkr(|tkr$|S|StD]*\}}|\}}||r0||r0|Sq0|tkrh|S|S)z@Lookup the canonical cloud-id for a given cloud_name and region.)rCLOUD_ID_REGION_PREFIX_MAPrDrH)rrrprefixZ cloud_id_testrZ valid_cloudr r r#r%s rTcCsj|sdSt|tr|St|tr*t|St|trV|dkrNt|dddStdtdt |dS)aLdata: a loaded object (strings, arrays, dicts). return something suitable for cloudinit vendordata_raw. if data is: None: return None string: return string list: return data the list is then processed in UserDataProcessor dict: return convert_vendordata(data.get('cloud-init')) NTz cloud-initF)recursez'vendordata['cloud-init'] cannot be dictz$Unknown data type for vendordata: %s) rGr8r rBrCrJconvert_vendordatarUrr)datar$r r r#r%8s     r%c@s eZdZdS)BrokenMetadataNr<r r r r#r'Psr'cCs4g}t|}|D]\}}|t|kr||q|Sr*)rrF)rrZret_listZdepsetrZdepsr r r#list_from_depends[s    r()rXfnamer)cCstzt|}Wn$tk r2ttd|YdSXztj||dddWn$tk rnttd|YdSXdS)z[Use pickle to serialize Datasource to a file as a cache. :return: True on success zFailed pickling datasource %sFwb)Zomoderz Failed pickling datasource to %sT)pickledumpsrrrr}r)rXr)Z pk_contentsr r r#rdsr)r)r)c Csd}ztj|dd}Wn<tk rR}ztj|rBtd||W5d}~XYnX|s\dSz t |WSt k r~YdStk rt td|YdSXdS)zBUse pickle to deserialize a instance Datasource from a cache file.NF)decodezfailed loading pickle in %s: %sz#Failed loading pickled blob from %s) rZ load_filerrrYisfiler}rr,rr9r)r)Zpickle_contentsrr r r#pkl_loadvs   r0)r>r )r)T)RabcrBrrr, collectionsrenumrrtypingrrrrr Z cloudinitr r r Zloggingr rrrvrZcloudinit.atomic_helperrZcloudinit.distrosrZcloudinit.eventrrZcloudinit.filtersrZcloudinit.helpersrZcloudinit.persistencerZcloudinit.reportingrZDSMODE_DISABLEDZ DSMODE_LOCALrZ DSMODE_PASSrZDEP_FILESYSTEMrrrZREDACT_SENSITIVE_VALUErrrZ getLoggerr0r}r"r&rr9r;r=rKrZr[r]ABCMetarrr8rrrr!rr%IOErrorr'r(rr0r r r r# s                #    %