Microsoft Azure Spark Client is a powerful tool that allows users to easily interact with Apache Spark through the Azure cloud platform. With this client, users can efficiently execute Spark jobs, analyze large datasets, and build data processing pipelines. The client provides a user-friendly interface that simplifies the process of writing and executing Spark applications, making it accessible to users of all skill levels.
swagger: '2.0'
info:
version: 2019-11-01-preview
title: Microsoft Azure SparkClient
x-ms-code-generation-settings:
useDateTimeOffset: true
x-ms-parameterized-host:
hostTemplate: '{endpoint}/livyApi/versions/{livyApiVersion}/sparkPools/{sparkPoolName}'
useSchemePrefix: false
parameters:
- $ref: '#/parameters/Endpoint'
- $ref: '#/parameters/LivyApiVersion'
- $ref: '#/parameters/SparkPoolName'
schemes:
- https
consumes:
- application/json
produces:
- application/json
paths:
/batches:
get:
tags:
- SparkBatch
description: >-
List all spark batch jobs which are running under a particular spark
pool.
operationId: microsoftAzureSparkbatchGetsparkbatchjobs
x-ms-examples:
List all spark batch jobs:
$ref: ./examples/SparkFrontend_SparkBatch_List.json
consumes:
- application/json
produces:
- application/json
parameters:
- name: from
in: query
description: Optional param specifying which index the list should begin from.
required: false
type: integer
format: int32
- name: size
in: query
description: "Optional param specifying the size of the returned list.\r\n By default it is 20 and that is the maximum."
required: false
type: integer
format: int32
- name: detailed
in: query
description: >-
Optional query param specifying whether detailed response is
returned beyond plain livy.
required: false
type: boolean
responses:
'200':
description: Success
schema:
$ref: '#/definitions/SparkBatchJobCollection'
summary: Microsoft Azure Get Batches
post:
tags:
- SparkBatch
description: Create new spark batch job.
operationId: microsoftAzureSparkbatchCreatesparkbatchjob
x-ms-examples:
Create new spark batch job.:
$ref: ./examples/SparkFrontend_SparkBatch_Create.json
consumes:
- application/json
produces:
- application/json
parameters:
- name: detailed
in: query
description: >-
Optional query param specifying whether detailed response is
returned beyond plain livy.
required: false
type: boolean
- name: sparkBatchJobOptions
in: body
description: Livy compatible batch job request payload.
required: true
schema:
$ref: '#/definitions/SparkBatchJobOptions'
responses:
'200':
description: Success
schema:
$ref: '#/definitions/SparkBatchJob'
summary: Microsoft Azure Post Batches
/batches/{batchId}:
get:
tags:
- SparkBatch
description: Gets a single spark batch job.
operationId: microsoftAzureSparkbatchGetsparkbatchjob
x-ms-examples:
Gets a single spark batch job.:
$ref: ./examples/SparkFrontend_SparkBatch_Get.json
consumes:
- application/json
produces:
- application/json
parameters:
- name: batchId
in: path
description: Identifier for the batch job.
required: true
type: integer
format: int32
- name: detailed
in: query
description: >-
Optional query param specifying whether detailed response is
returned beyond plain livy.
required: false
type: boolean
responses:
'200':
description: Success
schema:
$ref: '#/definitions/SparkBatchJob'
summary: Microsoft Azure Get Batches Batchid
delete:
tags:
- SparkBatch
description: Cancels a running spark batch job.
operationId: microsoftAzureSparkbatchCancelsparkbatchjob
x-ms-examples:
Cancels a running spark batch job.:
$ref: ./examples/SparkFrontend_SparkBatch_Delete.json
consumes:
- application/json
produces:
- application/json
parameters:
- name: batchId
in: path
description: Identifier for the batch job.
required: true
type: integer
format: int32
responses:
'200':
description: Success
summary: Microsoft Azure Delete Batches Batchid
/sessions:
get:
tags:
- SparkSession
description: List all spark sessions which are running under a particular spark pool.
operationId: microsoftAzureSparksessionGetsparksessions
x-ms-examples:
List all spark session jobs:
$ref: ./examples/SparkFrontend_SparkSession_List.json
consumes:
- application/json
produces:
- application/json
parameters:
- name: from
in: query
description: Optional param specifying which index the list should begin from.
required: false
type: integer
format: int32
- name: size
in: query
description: "Optional param specifying the size of the returned list.\r\n By default it is 20 and that is the maximum."
required: false
type: integer
format: int32
- name: detailed
in: query
description: >-
Optional query param specifying whether detailed response is
returned beyond plain livy.
required: false
type: boolean
responses:
'200':
description: Success
schema:
$ref: '#/definitions/SparkSessionCollection'
summary: Microsoft Azure Get Sessions
post:
tags:
- SparkSession
description: Create new spark session.
operationId: microsoftAzureSparksessionCreatesparksession
x-ms-examples:
Create new spark session.:
$ref: ./examples/SparkFrontend_SparkSession_Create.json
consumes:
- application/json
produces:
- application/json
parameters:
- name: detailed
in: query
description: >-
Optional query param specifying whether detailed response is
returned beyond plain livy.
required: false
type: boolean
- name: SparkSessionOptions
in: body
description: Livy compatible batch job request payload.
required: true
schema:
$ref: '#/definitions/SparkSessionOptions'
responses:
'200':
description: Success
schema:
$ref: '#/definitions/SparkSession'
summary: Microsoft Azure Post Sessions
/sessions/{sessionId}:
get:
tags:
- SparkSession
description: Gets a single spark session.
operationId: microsoftAzureSparksessionGetsparksession
x-ms-examples:
Gets a single spark session.:
$ref: ./examples/SparkFrontend_SparkSession_Get.json
consumes:
- application/json
produces:
- application/json
parameters:
- name: sessionId
in: path
description: Identifier for the session.
required: true
type: integer
format: int32
- name: detailed
in: query
description: >-
Optional query param specifying whether detailed response is
returned beyond plain livy.
required: false
type: boolean
responses:
'200':
description: Success
schema:
$ref: '#/definitions/SparkSession'
summary: Microsoft Azure Get Sessions Sessionid
delete:
tags:
- SparkSession
description: Cancels a running spark session.
operationId: microsoftAzureSparksessionCancelsparksession
x-ms-examples:
Cancels a running spark session.:
$ref: ./examples/SparkFrontend_SparkSession_Delete.json
consumes:
- application/json
produces:
- application/json
parameters:
- name: sessionId
in: path
description: Identifier for the session.
required: true
type: integer
format: int32
responses:
'200':
description: Success
summary: Microsoft Azure Delete Sessions Sessionid
/sessions/{sessionId}/reset-timeout:
put:
tags:
- SparkSession
description: >-
Sends a keep alive call to the current session to reset the session
timeout.
operationId: microsoftAzureSparksessionResetsparksessiontimeout
x-ms-examples:
Reset the session timeout.:
$ref: ./examples/SparkFrontend_SparkSession_ResetTimeout.json
consumes:
- application/json
produces:
- application/json
parameters:
- name: sessionId
in: path
description: Identifier for the session.
required: true
type: integer
format: int32
responses:
'200':
description: Success
summary: Microsoft Azure Put Sessions Sessionid Reset Timeout
/sessions/{sessionId}/statements:
get:
tags:
- SparkSession
description: Gets a list of statements within a spark session.
operationId: microsoftAzureSparksessionGetsparkstatements
x-ms-examples:
Gets a list of statements within a spark session.:
$ref: ./examples/SparkFrontend_SparkSession_ListStatements.json
consumes:
- application/json
produces:
- application/json
parameters:
- name: sessionId
in: path
description: Identifier for the session.
required: true
type: integer
format: int32
responses:
'200':
description: Success
schema:
$ref: '#/definitions/SparkStatementCollection'
summary: Microsoft Azure Get Sessions Sessionid Statements
post:
tags:
- SparkSession
description: Create statement within a spark session.
operationId: microsoftAzureSparksessionCreatesparkstatement
x-ms-examples:
Create statement within a spark session.:
$ref: ./examples/SparkFrontend_SparkSession_CreateStatement.json
consumes:
- application/json
produces:
- application/json
parameters:
- name: sessionId
in: path
description: Identifier for the session.
required: true
type: integer
format: int32
- name: SparkStatementOptions
in: body
description: Livy compatible batch job request payload.
required: true
schema:
$ref: '#/definitions/SparkStatementOptions'
responses:
'200':
description: Success
schema:
$ref: '#/definitions/SparkStatement'
summary: Microsoft Azure Post Sessions Sessionid Statements
/sessions/{sessionId}/statements/{statementId}:
get:
tags:
- SparkSession
description: Gets a single statement within a spark session.
operationId: microsoftAzureSparksessionGetsparkstatement
x-ms-examples:
Gets a single statement within a spark session.:
$ref: ./examples/SparkFrontend_SparkSession_GetStatement.json
consumes:
- application/json
produces:
- application/json
parameters:
- name: sessionId
in: path
description: Identifier for the session.
required: true
type: integer
format: int32
- name: statementId
in: path
description: Identifier for the statement.
required: true
type: integer
format: int32
responses:
'200':
description: Success
schema:
$ref: '#/definitions/SparkStatement'
summary: Microsoft Azure Get Sessions Sessionid Statements Statementid
/sessions/{sessionId}/statements/{statementId}/cancel:
post:
tags:
- SparkSession
description: Kill a statement within a session.
operationId: microsoftAzureSparksessionCancelsparkstatement
x-ms-examples:
Kill a statement within a session.:
$ref: ./examples/SparkFrontend_SparkSession_DeleteStatement.json
consumes:
- application/json
produces:
- application/json
parameters:
- name: sessionId
in: path
description: Identifier for the session.
required: true
type: integer
format: int32
- name: statementId
in: path
description: Identifier for the statement.
required: true
type: integer
format: int32
responses:
'200':
description: Success
schema:
$ref: '#/definitions/SparkStatementCancellationResult'
summary: Microsoft Azure Post Sessions Sessionid Statements Statementid Cancel
definitions:
SparkBatchJobCollection:
type: object
properties:
from:
format: int32
type: integer
description: The start index of fetched sessions.
total:
format: int32
type: integer
description: Number of sessions fetched.
sessions:
uniqueItems: false
type: array
description: Batch list
items:
$ref: '#/definitions/SparkBatchJob'
required:
- from
- total
description: Response for batch list operation.
SparkBatchJob:
type: object
properties:
livyInfo:
$ref: '#/definitions/SparkBatchJobState'
name:
type: string
description: The batch name.
workspaceName:
type: string
description: The workspace name.
sparkPoolName:
type: string
description: The Spark pool name.
submitterName:
type: string
description: The submitter name.
submitterId:
type: string
description: The submitter identifier.
artifactId:
type: string
description: The artifact identifier.
jobType:
enum:
- SparkBatch
- SparkSession
x-ms-enum:
name: SparkJobType
modelAsString: true
type: string
description: The job type.
result:
enum:
- Uncertain
- Succeeded
- Failed
- Cancelled
x-ms-enum:
name: SparkBatchJobResultType
modelAsString: true
type: string
description: The Spark batch job result.
schedulerInfo:
x-ms-client-name: scheduler
$ref: '#/definitions/SparkScheduler'
description: The scheduler information.
pluginInfo:
x-ms-client-name: plugin
$ref: '#/definitions/SparkServicePlugin'
description: The plugin information.
errorInfo:
x-ms-client-name: errors
uniqueItems: false
type: array
items:
$ref: '#/definitions/SparkServiceError'
description: The error information.
tags:
type: object
additionalProperties:
type: string
description: The tags.
id:
format: int32
type: integer
description: The session Id.
appId:
type: string
description: The application id of this session
x-nullable: true
appInfo:
type: object
additionalProperties:
type: string
description: The detailed application info.
x-nullable: true
state:
type: string
description: The batch state
log:
x-ms-client-name: logLines
uniqueItems: false
type: array
items:
type: string
description: The log lines.
x-nullable: true
required:
- id
SparkBatchJobState:
type: object
properties:
notStartedAt:
format: date-time
type: string
description: the time that at which "not_started" livy state was first seen.
x-nullable: true
startingAt:
format: date-time
type: string
description: the time that at which "starting" livy state was first seen.
x-nullable: true
runningAt:
format: date-time
type: string
description: the time that at which "running" livy state was first seen.
x-nullable: true
deadAt:
format: date-time
type: string
description: time that at which "dead" livy state was first seen.
x-nullable: true
successAt:
format: date-time
type: string
description: the time that at which "success" livy state was first seen.
x-nullable: true
killedAt:
x-ms-client-name: terminatedAt
format: date-time
type: string
description: the time that at which "killed" livy state was first seen.
x-nullable: true
recoveringAt:
format: date-time
type: string
description: the time that at which "recovering" livy state was first seen.
x-nullable: true
currentState:
type: string
description: the Spark job state.
jobCreationRequest:
$ref: '#/definitions/SparkRequest'
SparkScheduler:
type: object
properties:
submittedAt:
format: date-time
type: string
x-nullable: true
scheduledAt:
format: date-time
type: string
x-nullable: true
endedAt:
format: date-time
type: string
x-nullable: true
cancellationRequestedAt:
format: date-time
type: string
x-nullable: true
currentState:
enum:
- Queued
- Scheduled
- Ended
x-ms-enum:
name: SchedulerCurrentState
modelAsString: true
type: string
SparkServicePlugin:
type: object
properties:
preparationStartedAt:
format: date-time
type: string
x-nullable: true
resourceAcquisitionStartedAt:
format: date-time
type: string
x-nullable: true
submissionStartedAt:
format: date-time
type: string
x-nullable: true
monitoringStartedAt:
format: date-time
type: string
x-nullable: true
cleanupStartedAt:
format: date-time
type: string
x-nullable: true
currentState:
enum:
- Preparation
- ResourceAcquisition
- Queued
- Submission
- Monitoring
- Cleanup
- Ended
x-ms-enum:
name: PluginCurrentState
modelAsString: true
type: string
SparkServiceError:
type: object
properties:
message:
type: string
errorCode:
type: string
source:
enum:
- System
- User
- Unknown
- Dependency
x-ms-enum:
name: SparkErrorSource
modelAsString: true
type: string
SparkRequest:
type: object
properties:
name:
type: string
file:
type: string
className:
type: string
args:
x-ms-client-name: arguments
uniqueItems: false
type: array
items:
type: string
jars:
uniqueItems: false
type: array
items:
type: string
pyFiles:
x-ms-client-name: pythonFiles
uniqueItems: false
type: array
items:
type: string
files:
uniqueItems: false
type: array
items:
type: string
archives:
uniqueItems: false
type: array
items:
type: string
conf:
x-ms-client-name: configuration
type: object
additionalProperties:
type: string
driverMemory:
type: string
driverCores:
format: int32
type: integer
executorMemory:
type: string
executorCores:
format: int32
type: integer
numExecutors:
x-ms-client-name: executorCount
format: int32
type: integer
SparkBatchJobOptions:
type: object
properties:
tags:
type: object
additionalProperties:
type: string
artifactId:
type: string
name:
type: string
file:
type: string
className:
type: string
args:
x-ms-client-name: arguments
uniqueItems: false
type: array
items:
type: string
jars:
uniqueItems: false
type: array
items:
type: string
pyFiles:
x-ms-client-name: pythonFiles
uniqueItems: false
type: array
items:
type: string
files:
uniqueItems: false
type: array
items:
type: string
archives:
uniqueItems: false
type: array
items:
type: string
conf:
x-ms-client-name: configuration
type: object
additionalProperties:
type: string
driverMemory:
type: string
driverCores:
format: int32
type: integer
executorMemory:
type: string
executorCores:
format: int32
type: integer
numExecutors:
x-ms-client-name: executorCount
format: int32
type: integer
required:
- name
- file
SparkSessionCollection:
type: object
properties:
from:
format: int32
type: integer
total:
format: int32
type: integer
sessions:
uniqueItems: false
type: array
items:
$ref: '#/definitions/SparkSession'
required:
- from
- total
SparkSession:
type: object
properties:
livyInfo:
$ref: '#/definitions/SparkSessionState'
name:
type: string
workspaceName:
type: string
sparkPoolName:
type: string
submitterName:
type: string
submitterId:
type: string
artifactId:
type: string
jobType:
enum:
- SparkBatch
- SparkSession
x-ms-enum:
name: SparkJobType
modelAsString: true
type: string
result:
enum:
- Uncertain
- Succeeded
- Failed
- Cancelled
x-ms-enum:
name: SparkSessionResultType
modelAsString: true
type: string
schedulerInfo:
x-ms-client-name: scheduler
$ref: '#/definitions/SparkScheduler'
pluginInfo:
x-ms-client-name: plugin
$ref: '#/definitions/SparkServicePlugin'
errorInfo:
x-ms-client-name: errors
uniqueItems: false
type: array
items:
$ref: '#/definitions/SparkServiceError'
tags:
type: object
additionalProperties:
type: string
id:
format: int32
type: integer
appId:
type: string
x-nullable: true
appInfo:
type: object
additionalProperties:
type: string
x-nullable: true
state:
type: string
log:
x-ms-client-name: logLines
uniqueItems: false
type: array
items:
type: string
x-nullable: true
required:
- id
SparkSessionState:
type: object
properties:
notStartedAt:
format: date-time
type: string
x-nullable: true
startingAt:
format: date-time
type: string
x-nullable: true
idleAt:
format: date-time
type: string
x-nullable: true
deadAt:
format: date-time
type: string
x-nullable: true
shuttingDownAt:
format: date-time
type: string
x-nullable: true
killedAt:
x-ms-client-name: terminatedAt
format: date-time
type: string
x-nullable: true
recoveringAt:
format: date-time
type: string
x-nullable: true
busyAt:
format: date-time
type: string
x-nullable: true
errorAt:
format: date-time
type: string
x-nullable: true
currentState:
type: string
jobCreationRequest:
$ref: '#/definitions/SparkRequest'
SparkSessionOptions:
type: object
properties:
tags:
type: object
additionalProperties:
type: string
artifactId:
type: string
name:
type: string
file:
type: string
className:
type: string
args:
x-ms-client-name: arguments
uniqueItems: false
type: array
items:
type: string
jars:
uniqueItems: false
type: array
items:
type: string
pyFiles:
x-ms-client-name: pythonFiles
uniqueItems: false
type: array
items:
type: string
files:
uniqueItems: false
type: array
items:
type: string
archives:
uniqueItems: false
type: array
items:
type: string
conf:
x-ms-client-name: configuration
type: object
additionalProperties:
type: string
driverMemory:
type: string
driverCores:
format: int32
type: integer
executorMemory:
type: string
executorCores:
format: int32
type: integer
numExecutors:
x-ms-client-name: executorCount
format: int32
type: integer
required:
- name
SparkStatementCollection:
type: object
properties:
total_statements:
x-ms-client-name: total
format: int32
type: integer
statements:
uniqueItems: false
type: array
items:
$ref: '#/definitions/SparkStatement'
required:
- total_statements
SparkStatement:
type: object
properties:
id:
format: int32
type: integer
code:
type: string
state:
type: string
output:
$ref: '#/definitions/SparkStatementOutput'
x-nullable: true
required:
- id
SparkStatementOutput:
type: object
properties:
status:
type: string
execution_count:
format: int32
type: integer
data:
type: object
ename:
x-ms-client-name: errorName
type: string
x-nullable: true
evalue:
x-ms-client-name: errorValue
type: string
x-nullable: true
traceback:
uniqueItems: false
type: array
x-nullable: true
items:
type: string
required:
- execution_count
SparkStatementOptions:
type: object
properties:
code:
type: string
kind:
enum:
- spark
- pyspark
- dotnetspark
- sql
x-ms-enum:
name: SparkStatementLanguageType
modelAsString: true
values:
- name: Spark
value: spark
- name: PySpark
value: pyspark
- name: DotNetSpark
value: dotnetspark
- name: Sql
value: sql
type: string
SparkStatementCancellationResult:
type: object
properties:
msg:
type: string
x-ms-client-name: message
description: The msg property from the Livy API. The value is always "canceled".
parameters:
Endpoint:
name: endpoint
description: >-
The workspace development endpoint, for example
`https://myworkspace.dev.azuresynapse.net`.
required: true
type: string
in: path
x-ms-skip-url-encoding: true
x-ms-parameter-location: client
LivyApiVersion:
name: livyApiVersion
in: path
description: Valid api-version for the request.
required: true
type: string
x-ms-client-default: 2019-11-01-preview
x-ms-skip-url-encoding: true
x-ms-parameter-location: client
SparkPoolName:
name: sparkPoolName
in: path
description: Name of the spark pool.
required: true
type: string
x-ms-skip-url-encoding: true
x-ms-parameter-location: client
tags:
- name: SparkBatch
- name: SparkSession