{"type":"doc","content":[{"type":"heading","attrs":{"level":1},"content":[{"text":"Step 1) Required Resources","type":"text"}]},{"type":"paragraph","content":[{"text":"Lakehouse Optimizer requires the following resources to already be created:","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://blueprinttechnologies.atlassian.net/wiki/spaces/BLMPD/pages/2577662006"}}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Step 2) Configuration Prerequisites","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"AWS Secrets Manager","type":"text","marks":[{"type":"strong"}]},{"text":" needs to be configured with the following secret key value pairs. Suggested name for the secret is ‘bplm-credentials’:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"storage-access-key","type":"text","marks":[{"type":"code"}]},{"text":" - Optional, AWS Access Key used for accessing Amazon DynamoDB and Amazon SQS by the telemetry agent","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"storage-secret-key","type":"text","marks":[{"type":"code"}]},{"text":" - Optional, AWS Secret Key used for accessing Amazon DynamoDB and Amazon SQS by the telemetry agent","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"underline"},{"type":"strong"}]},{"text":" DynamoDB is the telemetry data store, access from the LHO services or telemetry agents in Databricks workspaces can be enabled either with the access key/secret key pair or via IAM Roles/Credentials and Instance Profiles, in which case the key pair above becomes optional","type":"text"}]}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"service-account-username","type":"text","marks":[{"type":"code"}]},{"text":" - Databricks service account username","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"service-account-password","type":"text","marks":[{"type":"code"}]},{"text":" - Databricks service account password","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"underline"},{"type":"strong"}]},{"text":" the Databricks service account is required for access to the Billable Usage Logs of Databricks Accounts API. These logs will fuel all the consumption reports at the Databricks account, workspace, job, job run, task run, nested notebook, cluster, notebook, dlt pipeline, dlt update etc level in the tool and it will help prioritize optimization efforts.","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":" It is a Databricks Account user with username and password, since the only authentication supported by the Databricks Accounts API is username/password.","type":"text"}]}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"mssql-password","type":"text","marks":[{"type":"code"}]},{"text":" - SQL Login password for the SQL Database","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"application-encryption-secret","type":"text","marks":[{"type":"code"}]},{"text":" - encryption key for storing PATs (Personal Access Tokens) and the Databricks Accounts credentials (billable usage logs) in the LHO SQL database","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"msft-provider-auth-secret","type":"text","marks":[{"type":"code"}]},{"text":" - Optional, Client secret value from azure app registration","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"underline"},{"type":"strong"}]},{"text":" The Service Principal secret key is needed in case you want LHO configured with Azure Active Directory for login and SSL. If you choose to use Databricks authentication only this is not needed and the secret can be omitted.","type":"text"}]}]}]}]}]}]}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Step 3) Installation procedure","type":"text"}]},{"type":"paragraph","content":[{"text":"1. ","type":"text","marks":[{"type":"strong"}]},{"text":"SSH into the BPLM VM configured at ","type":"text"},{"text":"Step 1) Required Resources","type":"text","marks":[{"type":"em"}]},{"text":".","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":" ","type":"text"},{"text":"Download ","type":"text","marks":[{"type":"strong"}]},{"text":"the install archive by running the following command:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Ubuntu:","type":"text"},{"type":"hardBreak"},{"text":"wget https://bplmdemoappstg.blob.core.windows.net/deployment/vm-aws/ubuntu.zip","type":"text","marks":[{"type":"code"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"CentOS:","type":"text"},{"type":"hardBreak"},{"text":"wget https://bplmdemoappstg.blob.core.windows.net/deployment/vm-aws/centos.zip","type":"text","marks":[{"type":"code"}]}]}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Extract ","type":"text","marks":[{"type":"strong"}]},{"text":"the archive contents","type":"text"},{"type":"hardBreak"},{"text":"unzip ubuntu.zip","type":"text","marks":[{"type":"code"}]},{"text":" / ","type":"text"},{"text":"unzip centos.zip","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"paragraph","marks":[{"type":"indentation","attrs":{"level":1}}],"content":[{"text":"In the destination directory you should see the following files:","type":"text"}]},{"type":"codeBlock","content":[{"text":" .env\n docker-compose.yml\n setup.sh\n start.sh","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Before you start setup you need to fill out the .env file with the required information). Open the file in your editor of choice and fill in the values.","type":"text"}]},{"type":"orderedList","attrs":{"order":1},"content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Please find a brief explanation of the .env values below","type":"text"}]}]}]}]}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Lakehouse uses databricks authentication currently with an optional setup of Azure AD as the identity provider. If you will not be using AAD, you do not need to fill out ","type":"text"},{"text":"SERVICE_PRINCIPAL_CLIENTID","type":"text","marks":[{"type":"code"}]},{"text":", ","type":"text"},{"text":"SERVICE_PRINCIPAL_TENANTID","type":"text","marks":[{"type":"code"}]},{"text":",or ","type":"text"},{"text":"SERVICE_PRINCIPAL_CLIENTSECRET","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":"In this case you must remove “","type":"text"},{"text":",active-directory\"","type":"text","marks":[{"type":"code"}]},{"text":" from ","type":"text"},{"text":"AUTHENTICATION_PROVIDER","type":"text","marks":[{"type":"code"}]}]}]},{"type":"codeBlock","content":[{"text":"LOG_LEVEL=info\nLOG_LEVEL_APP=info\nLOG_LEVEL_HTTP_HEADERS=error\n\nAPPSERVICE_URL=\n\nSQL_DATABASE=master\nSQL_SERVER_HOST=\nSQL_USER=\n\nSTORAGE_AWS_REGION=\nSTORAGE_AWS_TABLE_PREFIX=bplm\n\nAWS_SECRETS_MANAGER_ENABLED=true\nAWS_SECRETS_MANAGER_REGION=\nBPLM_SECRET_NAME=\nSERVER_SSL_ENABLED=true\nSERVER_SSL_KEY-STORE=/keystore/bplm.p12\nSERVER_SSL_KEY-STORE-PASSWORD=\nSERVER_SSL_KEY-STORE-TYPE=PKCS12\nSERVER_SSL_KEY-ALIAS=bplm\nSERVER_SSL_KEY-PASSWORD=\n\nSERVICE_PRINCIPAL_CLIENTID=\nSERVICE_PRINCIPAL_TENANTID=\n#SERVICE_PRINCIPAL_CLIENTSECRET=${msft-provider-auth-secret}\n\nDATABRICKS_ACCOUNT_ID=\nDATARICKS_SERVICE_PRINCIPAL=\nACTIVE-DIRECTORY_HOST=https://login.microsoftonline.com\nACTIVE-DIRECTORY_TOKEN-ENDPOINT=/oauth2/v2.0/token\nACTIVE-DIRECTORY_AUTHORIZE-ENDPOINT=/oauth2/v2.0/authorize\nACTIVE-DIRECTORY_JWK-ENDPOINT=/discovery/keys\nACTIVE-DIRECTORY_USER-INFO-URI=https://graph.microsoft.com/oidc/userinfo\n\nCLOUD_PROVIDER=AWS\nAUTHENTICATION_PROVIDER=databricks-account,active-directory\nSPRING_PROFILES_ACTIVE=production-aws\nADMIN_APP_ROLE=bplm-admin\nMETRIC_PROCESSOR_ENABLED=true\nSTORAGE_THROUGH_IAM_CREDENTIALS=true\n#metric.queueMonitoring.compactionTimeout=PT25M\nAPPLICATION_NOTIFICATION_JOBNOTIFICATIONQUEUENAME=\n\n#CONSUMPTION_BILLABLE_USAGE_PATH=s3a://{{s3-bucket}}/dbx-costs/billable-usage/csv\n\n#CROSS_ACCOUNT_ASSUME_IAM_ROLE_AGENT=\n#CROSS_ACCOUNT_ASSUME_IAM_ROLE_S3_DBX_BILLING_APP=\n#CROSS_ACCOUNT_ASSUME_IAM_ROLE_DYNAMO_SQS_APP=\n#CROSS_ACCOUNT_ASSUME_IAM_ROLE_COST_EXPLORER_APP=\n#CROSS_ACCOUNT_ASSUME_IAM_ROLE_TAG_WORKSPACE_RESOURCE_APP=\n\n# Configuration example from cross account assume roles\n#CROSS_ACCOUNT_ASSUME_IAM_ROLE_COST_EXPLORER_APP=arn:aws:iam:::role/bplm-dev-costexplorer-role,arn:aws:iam::153067919175:role/examplempl-of-xaccount-permission-role-for-cost-explorer-and-tagz\n#CROSS_ACCOUNT_ASSUME_IAM_ROLE_TAG_WORKSPACE_RESOURCE_APP=arn:aws:iam:::role/vt-bplm-test-multi-aws-acc-tags,arn:aws:iam::153067919175:role/example-of-xaccount-permission-role-for-cost-explorer-and-tagz\n#CROSS_ACCOUNT_ASSUME_IAM_ROLE_DYNAMO_SQS_APP=arn:aws:iam:::role/bplm-dev-dynamosqs-role\n#CROSS_ACCOUNT_ASSUME_IAM_ROLE_AGENT=arn:aws:iam:::role/bplm-dev-dynamosqs-collector-role\n#CROSS_ACCOUNT_ASSUME_IAM_ROLE_S3_DBX_BILLING_APP=arn:aws:iam::153067919175:role/xaccount-s3-accesss-role","type":"text"}]},{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"underline"},{"type":"strong"}]},{"text":" due to the docker version provided by CentOS the ","type":"text"},{"text":"SERVICE_PRINCIPAL_CLIENTSECRET","type":"text","marks":[{"type":"code"}]},{"text":" can not be pulled from the secrets manager.","type":"text"}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"4.","type":"text","marks":[{"type":"strong"}]},{"text":" Run ./setup.sh providing the domain you wish to create an SSL cert for, the version of the Lakehouse Optimizer, and an admin email that will be used to configure certbot’s notifications when creating an SSL certificate.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"If you do not currently have a registered DNS entry for the Lakehouse Optimizer, you can skip setting up SSL certs by not supplying the ","type":"text"},{"text":"cert_domain","type":"text","marks":[{"type":"code"}]},{"text":" or ","type":"text"},{"text":"email_certbot","type":"text","marks":[{"type":"code"}]},{"text":" arguments.","type":"text"}]}]},{"type":"codeBlock","attrs":{"language":"shell"},"content":[{"text":"eg: ./setup.sh --cert_domain \"lakehouse-monitor.company.com\" --version 2.5.0 --email_certbot notifications@company.com","type":"text"}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"5.","type":"text","marks":[{"type":"strong"}]},{"text":" After the setup script completes, run start.sh to pull down the application container and start it","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"For being able to run the start script you will need to provide the Blueprint Docker Registry username and password that will be used by docker to pull the BPLM images from the Blueprint container registry: ","type":"text"},{"text":"bplm-acr-token / ","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"codeBlock","attrs":{"language":"shell"},"content":[{"text":"eg: ./start.sh example-acr-user providedPassword","type":"text"}]},{"type":"paragraph"}],"version":1}

Browser not supported