DataApps

import os

from utils.rc.client.requests import Requests
from utils.rc.client.auth import AuthClient

from utils.rc.dtos.project import Project
from utils.rc.dtos.dataset import Dataset
from utils.rc.dtos.recipe import Recipe
from utils.rc.dtos.transform import Transform
from utils.rc.dtos.template import Template
from utils.rc.dtos.template import TemplateTransform
from utils.rc.dtos.template import TemplateInput
from utils.rc.dtos.artifact import Artifact
from utils.rcclient.libs.dataapp_generator.dataapp_generator import DataappGenerator
from utils.rcclient.entities.dataapp import DataappParamMetadata

from utils.rc.dtos.template_v2 import TemplateV2, TemplateTransformV2

import requests
import pandas as pd
import logging
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)

Artifact.add_file("artifact-id", "data/employee_promotion_case.csv")
Artifact.add_file("artifact-id2", "data/Region_States.csv")

2023-01-31 03:33:48.242 INFO    utils.rcclient.entities.artifact: File successfully added
2023-01-31 03:33:49.364 INFO    utils.rcclient.entities.artifact: File successfully added

da1 = DataappGenerator.generate_app_template(name='testapp1', display_name="RC streamlit test app 1", path="dataapps")
da1.create_input_param(name="entity_param", metadata=DataappParamMetadata(input_name="Entity", is_required=True, default_value="entity"))
da1.create_input_param(name="artifact_param", metadata=DataappParamMetadata(input_name="Artifact", is_required=True, default_value="artifact"))
da1.publish(force=True)

da2 = DataappGenerator.generate_app_template(name='testapp2', display_name="RC streamlit test app 2", path="dataapps")
da2.create_input_param(name="entity_param", metadata=DataappParamMetadata(input_name="Entity", is_required=True, default_value="entity"))
da2.create_input_param(name="artifact_param", metadata=DataappParamMetadata(input_name="Artifact", is_required=True, default_value="artifact"))
da2.publish(force=True)

da3 = DataappGenerator.generate_app_template(name='testapp3', display_name="RC streamlit test app 3", path="dataapps")
da3.create_input_param(name="entity_param", metadata=DataappParamMetadata(input_name="Entity", is_required=True, default_value="entity"))
da3.create_input_param(name="artifact_param", metadata=DataappParamMetadata(input_name="Artifact", is_required=True, default_value="artifact"))
da3.publish(force=True)

2023-01-31 03:33:49.391 WARNING utils.rcclient.libs.dataapp_generator.dataapp_generator: App testapp1 is already generated!
2023-01-31 03:33:49.432 INFO    utils.rcclient.services.dataapp_service: Validating dataapp (testapp1)...
INFO:Searching streamlit==1.15.0,,--extra-index-url...
INFO:Found ✓
INFO:Searching https://us-central1-python.pkg.dev/rapidcanvas-361003/pypi/simple,utils==0.25.dev7,...
ERROR:No matching distribution found for https://us-central1-python.pkg.dev/rapidcanvas-361003/pypi/simple,utils==0.25.dev7,
2023-01-31 03:33:53.954 WARNING utils.rcclient.services.dataapp_service: Invalid distribution/s found!
2023-01-31 03:33:53.954 INFO    utils.rcclient.services.dataapp_service: Validation successful!
2023-01-31 03:33:55.141 INFO    utils.rcclient.services.dataapp_service: Updating dataapp testapp1
2023-01-31 03:33:55.208 INFO    utils.rcclient.services.dataapp_service: Dataapp testapp1 updated successfully!
2023-01-31 03:33:55.211 WARNING utils.rcclient.libs.dataapp_generator.dataapp_generator: App testapp2 is already generated!
2023-01-31 03:33:55.243 INFO    utils.rcclient.services.dataapp_service: Validating dataapp (testapp2)...
INFO:Searching streamlit==1.15.0,,--extra-index-url...
INFO:Found ✓
INFO:Searching https://us-central1-python.pkg.dev/rapidcanvas-361003/pypi/simple,utils==0.25.dev7,...
ERROR:No matching distribution found for https://us-central1-python.pkg.dev/rapidcanvas-361003/pypi/simple,utils==0.25.dev7,
2023-01-31 03:33:59.454 WARNING utils.rcclient.services.dataapp_service: Invalid distribution/s found!
2023-01-31 03:33:59.455 INFO    utils.rcclient.services.dataapp_service: Validation successful!
2023-01-31 03:34:00.576 INFO    utils.rcclient.services.dataapp_service: Updating dataapp testapp2
2023-01-31 03:34:00.627 INFO    utils.rcclient.services.dataapp_service: Dataapp testapp2 updated successfully!
2023-01-31 03:34:00.630 WARNING utils.rcclient.libs.dataapp_generator.dataapp_generator: App testapp3 is already generated!
2023-01-31 03:34:00.657 INFO    utils.rcclient.services.dataapp_service: Validating dataapp (testapp3)...
INFO:Searching streamlit==1.15.0,,--extra-index-url...
INFO:Found ✓
INFO:Searching https://us-central1-python.pkg.dev/rapidcanvas-361003/pypi/simple,utils==0.25.dev7,...
ERROR:No matching distribution found for https://us-central1-python.pkg.dev/rapidcanvas-361003/pypi/simple,utils==0.25.dev7,
2023-01-31 03:34:04.686 WARNING utils.rcclient.services.dataapp_service: Invalid distribution/s found!
2023-01-31 03:34:04.686 INFO    utils.rcclient.services.dataapp_service: Validation successful!
2023-01-31 03:34:05.809 INFO    utils.rcclient.services.dataapp_service: Updating dataapp testapp3
2023-01-31 03:34:05.864 INFO    utils.rcclient.services.dataapp_service: Dataapp testapp3 updated successfully!

project = Project.create(
    name="Example Dataapp",
    description="Testing Dataapp",
    createEmpty=True
)

2023-01-31 03:34:05.943 INFO    root: Found existing project by name: Example Dataapp
2023-01-31 03:34:05.944 INFO    root: Deleting existing project
2023-01-31 03:34:05.980 INFO    root: Creating new project by name: Example Dataapp

project.id

'3842b9be-b95f-4a03-ac1d-3ea88a5518e5'

titanic = project.addDataset(
    dataset_name="titanic",
    dataset_description="titanic golden",
    dataset_file_path="data/titanic.csv"
)

2023-01-31 03:34:06.270 INFO    root: Creating new dataset by name:titanic
2023-01-31 03:34:13.611 INFO    root: Uploading file data/titanic.csv ....
2023-01-31 03:34:29.926 INFO    root: Uploading Done

employee = project.addDataset(
    dataset_name="employee",
    dataset_description="employee golden",
    dataset_file_path="data/employee_promotion_case.csv"
)

2023-01-31 03:34:29.975 INFO    root: Creating new dataset by name:employee
2023-01-31 03:34:30.010 INFO    root: Uploading file data/employee_promotion_case.csv ....
2023-01-31 03:34:53.319 INFO    root: Uploading Done

# da1 = Dataapp.find_by_name("testapp1")
dataapp_params = {"entity_param": "titanic", "artifact_param": "artifact-id"}
project.install_dataapp(da3, metadata={}, dataapp_params=dataapp_params)

# Launching dataapp before using them in recipe
da1.launch()
da2.launch()
da3.launch()

create_chart_template = TemplateV2(
    name="create_chart", description="Create chart using custom data apps", project_id=project.id,
    source="CUSTOM", status="ACTIVE", tags=["UI", "Scalar"]
)
create_chart_template.publish("transforms/create_chart.ipynb")

2023-01-31 03:36:27.529 INFO    root: Publishing template | data=TemplateV2(name='create_chart', display_name=None, id=None, version='1.0', project_id='3842b9be-b95f-4a03-ac1d-3ea88a5518e5', projectId='3842b9be-b95f-4a03-ac1d-3ea88a5518e5', is_global=False, description='Create chart using custom data apps', tags=['UI', 'Scalar'], baseTransforms=[TemplateTransformV2(type='python', params={'notebookName': 'create_chart.ipynb'})], base_transforms=None, source='CUSTOM', status='ACTIVE', inputs=[])
2023-01-31 03:36:29.028 INFO    root: Template Published
2023-01-31 03:36:29.068 WARNING papermill.translators: Black is not installed, parameters wont be formatted
2023-01-31 03:36:29.070 WARNING papermill: Input notebook does not contain a cell with tag 'parameters'
2023-01-31 03:36:29.878 INFO    papermill: Executing notebook with kernel: python3
INFO:User authenticated successfully
INFO:Creating template input | nb_stage=COMPILE_TIME
INFO:Creating template input | nb_stage=COMPILE_TIME
INFO:Creating template input | nb_stage=COMPILE_TIME
INFO:Creating template input | nb_stage=COMPILE_TIME
INFO:
************************************************************************************
**    CREATING INPUTS: input_dataset, employee_dataset, chart_app1, chart_app2    **
************************************************************************************
2023-01-31 03:36:30.695 INFO    utils.rc.wrapper.templates: Inputs created successfully | template_id=a5bd65d9-14c1-44e9-aded-b89518f866a8
Inputs created successfully | template_id=a5bd65d9-14c1-44e9-aded-b89518f866a8

create_chart_transform = Transform()
create_chart_transform.templateId = create_chart_template.id
create_chart_transform.name='create_chart_transform'
create_chart_transform.variables = {
    'input_dataset': 'titanic',
    'employee_dataset': 'employee',
    'chart_app1': 'testapp1',
    'chart_app2': 'testapp2'
}

create_chart_recipe = project.addRecipe([titanic, employee], name='create_chart_recipe')
create_chart_recipe.add_transform(create_chart_transform)

2023-01-31 03:36:31.001 INFO    root: Creating new recipe
2023-01-31 03:36:31.044 INFO    root: Adding new transform
2023-01-31 03:39:37.497 INFO    root: Transform added Successfully

create_chart_recipe.run()

2023-01-31 03:39:37.570 INFO    root: Started running
2023-01-31 03:39:37.571 INFO    root: You can look at the progress on UI at http://localhost:8082/#/projects/3842b9be-b95f-4a03-ac1d-3ea88a5518e5
2023-01-31 03:39:45.951 INFO    root: No errors found

host = os.environ["RC_PYTHON_CLIENT_HOST"]
base_url = host.split("/api/")[0]
apps = ['testapp1', 'testapp2', 'testapp3']
for app in apps:
    url = f"{base_url}/dataapps/{app}-{Requests.getTenantId()}/healthz"
    res = requests.get(url)
    print(f"app: {app} | url={url} | status: {res.status_code}")
    assert res.status_code == 200

project.uninstall_dataapp(da3)
project.uninstall_dataapp(da2)
project.uninstall_dataapp(da1)