Lakehouses

generate_lakehouse_shortcuts_metadata(workspace, lakehouse)

Source code in src/pyfabricops/helpers/lakehouses.py

def generate_lakehouse_shortcuts_metadata(
    workspace: str, lakehouse: str
) -> Union[Dict[str, Any], None]:
    """ """
    # Create shortcuts.metadata.json
    shortcuts_list = list_shortcuts(workspace, lakehouse, df=False)

    if len(shortcuts_list) == 0:
        return None

    # Init a empty list for shortcuts
    shortcuts_list_new = []

    for shortcut_dict in shortcuts_list:
        shortcut_target = shortcut_dict['target']
        shortcut_target_type = (
            shortcut_target['type'][0].lower() + shortcut_target['type'][1:]
        )
        shortcut_target_workspace_id = shortcut_target[shortcut_target_type][
            'workspaceId'
        ]
        shortcut_target_item_id = shortcut_target[shortcut_target_type][
            'itemId'
        ]

        workspace_items = list_items(shortcut_target_workspace_id, df=False)
        for item in workspace_items:
            if item['id'] == shortcut_target_item_id:
                shortcut_target_item_type = item['type']
                break

    # Check if the workspace_id is equal shortcut_target_workspace_id then uuid zero
    if shortcut_target_workspace_id == resolve_workspace(workspace):
        shortcut_target_workspace_id = '00000000-0000-0000-0000-000000000000'

    # Create item type if not exists
    if 'artifactType' not in shortcut_dict['target'][shortcut_target_type]:
        shortcut_dict['target'][shortcut_target_type]['artifactType'] = ''
    if 'workspaceId' not in shortcut_dict['target'][shortcut_target_type]:
        shortcut_dict['target'][shortcut_target_type]['workspaceId'] = ''

    # Update if exists
    shortcut_dict['target']['oneLake'][
        'artifactType'
    ] = shortcut_target_item_type
    shortcut_dict['target']['oneLake'][
        'workspaceId'
    ] = shortcut_target_workspace_id

    shortcuts_list_new.append(shortcut_dict)

    return shortcuts_list_new

get_all_lakehouses_config(workspace)

Generate lakehouses config from a workspace.

Parameters:

Name	Type	Description	Default
workspace	str	The name or ID from the workspace.	required

Returns:

Type	Description
Union[Dict[str, Any], None]	The dict config from the lakehouses of the workspace

Source code in src/pyfabricops/helpers/lakehouses.py

def get_all_lakehouses_config(workspace: str) -> Union[Dict[str, Any], None]:
    """
    Generate lakehouses config from a workspace.

    Args:
        workspace (str): The name or ID from the workspace.

    Returns:
        (Union[Dict[str, Any], None]): The dict config from the lakehouses of the workspace
    """
    items = list_valid_lakehouses(workspace, df=False)

    if items is None:
        return None

    config = {}

    for item in items:

        item_data = get_lakehouse(workspace, item['id'], df=False)

        config[item['displayName']] = {
            'id': item['id'],
            'description': item.get('description', None),
            'folder_id': ''
            if item.get('folderId') is None or pd.isna(item.get('folderId'))
            else item['folderId'],
            'sql_endpoint_connection_string': item_data['properties'][
                'sqlEndpointProperties'
            ]['connectionString'],
            'sql_endpoint_id': item_data['properties'][
                'sqlEndpointProperties'
            ]['id'],
        }

    return config

get_lakehouse_config(workspace, lakehouse)

Get a specific lakehouse config from a workspace.

Parameters:

Name	Type	Description	Default
workspace	str	The name or ID from the workspace.	required
lakehouse	str	The name or ID from the lakehouse.	required

Returns:

Type	Description
Union[Dict[str, Any], None]	The dict config from the lakehouse

Source code in src/pyfabricops/helpers/lakehouses.py

def get_lakehouse_config(
    workspace: str, lakehouse: str
) -> Union[Dict[str, Any], None]:
    """
    Get a specific lakehouse config from a workspace.

    Args:
        workspace (str): The name or ID from the workspace.
        lakehouse (str): The name or ID from the lakehouse.

    Returns:
        (Union[Dict[str, Any], None]): The dict config from the lakehouse
    """
    item = lakehouse
    item_data = get_lakehouse(workspace, item, df=False)

    if item_data is None:
        return None

    else:
        config = {}
        config = config[item_data.get('displayName')] = {}

        config = {
            'id': item_data['id'],
            'description': item_data.get('description', None),
            'folder_id': ''
            if item_data.get('folderId') is None
            or pd.isna(item_data.get('folderId'))
            else item_data['folderId'],
            'sql_endpoint_connection_string': item_data.get(
                'properties_sqlEndpointProperties_connectionString'
            ),
            'sql_endpoint_id': item_data.get(
                'properties_sqlEndpointProperties_id'
            ),
        }

        return config

list_valid_lakehouses(workspace, df=True)

Generate a list of valid lakehouses from a workspace.

Parameters:

Name	Type	Description	Default
workspace	str	The name or ID from the workspace.	required

Returns:

Type	Description
Union[Dict[str, Any], None]	The list of valids lakehouses of the workspace

Source code in src/pyfabricops/helpers/lakehouses.py

@df
def list_valid_lakehouses(
    workspace: str,
    df: Optional[bool] = True,
) -> Union[DataFrame, List[Dict[str, Any]], None]:
    """
    Generate a list of valid lakehouses from a workspace.

    Args:
        workspace (str): The name or ID from the workspace.

    Returns:
        (Union[Dict[str, Any], None]): The list of valids lakehouses of the workspace
    """
    items = list_lakehouses(workspace)

    if items is None:
        return None

    return items[
        ~items['displayName'].str.contains('staging', case=False, na=False)
    ].to_dict(orient='records')

save_lakehouse_shortcuts_metadata(shortcuts_metadata, path)

Source code in src/pyfabricops/helpers/lakehouses.py

def save_lakehouse_shortcuts_metadata(
    shortcuts_metadata: Dict[str, Any], path: str
) -> None:
    """ """
    with open(Path(path) / 'shortcuts.metadata.json', 'w') as f:
        json.dump(shortcuts_metadata, f, indent=2)