[docs]classJinaConstructor(FullConstructor):"""Convert List into tuple when doing hashing."""
[docs]defget_hashable_key(self,key):""" Get the hash value of key. :param key: key value to be hashed. :return: Hash value of key. """try:hash(key)except:ifisinstance(key,list):foriinrange(len(key)):ifnotisinstance(key[i],collections.abc.Hashable):key[i]=self.get_hashable_key(key[i])key=tuple(key)returnkeyraiseValueError(f'unhashable key: {key}')returnkey
[docs]defconstruct_mapping(self,node,deep=True):""" Build the mapping from node. :param node: the node to traverse :param deep: required param from YAML constructor :return: Mapped data """ifisinstance(node,MappingNode):self.flatten_mapping(node)returnself._construct_mapping(node,deep=deep)
def_construct_mapping(self,node,deep=True):ifnotisinstance(node,MappingNode):raiseConstructorError(None,None,'expected a mapping node, but found %s'%node.id,node.start_mark,)mapping={}forkey_node,value_nodeinnode.value:key=self.construct_object(key_node,deep=True)ifnotisinstance(key,collections.abc.Hashable):try:key=self.get_hashable_key(key)exceptExceptionasexc:raiseConstructorError('while constructing a mapping',node.start_mark,'found unacceptable key (%s)'%exc,key_node.start_mark,)value=self.construct_object(value_node,deep=deep)mapping[key]=valuereturnmapping
[docs]classJinaResolver(Resolver):"""Remove `on|On|ON` as bool resolver."""pass
[docs]classJinaLoader(Reader,Scanner,Parser,Composer,JinaConstructor,JinaResolver):""" The Jina loader which should be able to load YAML safely. :param stream: the stream to load. """def__init__(self,stream):Reader.__init__(self,stream)Scanner.__init__(self)Parser.__init__(self)Composer.__init__(self)JinaConstructor.__init__(self)JinaResolver.__init__(self)
[docs]defparse_config_source(path:Union[str,TextIO,Dict],allow_stream:bool=True,allow_yaml_file:bool=True,allow_raw_yaml_content:bool=True,allow_class_type:bool=True,allow_dict:bool=True,allow_json:bool=True,extra_search_paths:Optional[List[str]]=None,*args,**kwargs,)->Tuple[TextIO,Optional[str]]:""" Check if the text or text stream is valid. .. # noqa: DAR401 :param path: the multi-kind source of the configs. :param allow_stream: flag :param allow_yaml_file: flag :param allow_raw_yaml_content: flag :param allow_class_type: flag :param allow_dict: flag :param allow_json: flag :param extra_search_paths: extra paths to search for :param args: unused :param kwargs: unused :return: a tuple, the first element is the text stream, the second element is the file path associate to it if available. """importioifnotpath:raiseBadConfigSourceelifallow_dictandisinstance(path,dict):fromjina.jamlimportJAMLtmp=JAML.dump(path)returnio.StringIO(tmp),Noneelifallow_streamandhasattr(path,'read'):# already a readable streamreturnpath,Noneelifallow_yaml_fileandis_yaml_filepath(path):comp_path=complete_path(path,extra_search_paths)returnopen(comp_path,encoding='utf8'),comp_pathelifallow_raw_yaml_contentandpath.lstrip().startswith(('!','jtype')):# possible YAML contentpath=path.replace('|','\n with: ')returnio.StringIO(path),Noneelifallow_class_typeandpath.isidentifier():# possible class namereturnio.StringIO(f'!{path}'),Noneelifallow_jsonandisinstance(path,str):try:fromjina.jamlimportJAMLtmp=json.loads(path)tmp=JAML.dump(tmp)returnio.StringIO(tmp),Noneexceptjson.JSONDecodeError:raiseBadConfigSource(path)else:raiseBadConfigSource(f'{path} can not be resolved, it should be a readable stream,'' or a valid file path, or a supported class name.')
[docs]defcomplete_path(path:str,extra_search_paths:Optional[List[str]]=None)->str:""" Complete the path of file via searching in abs and relative paths. :param path: path of file. :param extra_search_paths: extra paths to conduct search :return: Completed file path. """_p=_search_file_in_paths(path,extra_search_paths)if_pisNoneandos.path.exists(path):# this checks both abs and relative paths already_p=pathif_p:returnos.path.abspath(_p)else:raiseFileNotFoundError(f'can not find {path}')
def_search_file_in_paths(path,extra_search_paths:Optional[List[str]]=None):""" Search in all dirs of the PATH environment variable and all dirs of files used in the call stack. :param path: the path to search for :param extra_search_paths: any extra locations to search for :return: the path (if found) """importinspectsearch_paths=[]ifextra_search_paths:search_paths.extend((vforvinextra_search_paths))frame=inspect.currentframe()# iterates over the call stackwhileframe:search_paths.append(os.path.dirname(inspect.getfile(frame)))frame=frame.f_backsearch_paths+=os.environ['PATH'].split(os.pathsep)# return first occurrence of path. If it does not exist, return None.forpinsearch_paths:_p=os.path.join(p,path)ifos.path.exists(_p):return_p
[docs]defload_py_modules(d:Dict,extra_search_paths:Optional[List[str]]=None)->None:""" Find 'py_modules' in the dict recursively and then load them. :param d: the dictionary to traverse :param extra_search_paths: any extra paths to search """mod=[]def_finditem(obj,key='py_modules'):value=obj.get(key,[])ifisinstance(value,str):mod.append(value)elifisinstance(value,(list,tuple)):mod.extend(value)fork,vinobj.items():ifisinstance(v,dict):_finditem(v,key)_finditem(d)ifmod:iflen(mod)>1:warnings.warn('It looks like you are trying to import multiple python modules using'' `py_modules`. When using multiple python files to define an executor,'' the recommended practice is to structure the files in a python'' package, and only import the `__init__.py` file of that package.'' For more details, please check out the cookbook: ''https://docs.jina.ai/fundamentals/executor/repository-structure/')mod=[complete_path(m,extra_search_paths)forminmod]PathImporter.add_modules(*mod)