Overview
Getting Started
User Guides
API Reference
Development
Migration Guides
3.5.2
Spark SQL
Core Classes
pyspark.sql.SparkSession
pyspark.sql.Catalog
pyspark.sql.DataFrame
pyspark.sql.Column
pyspark.sql.Observation
pyspark.sql.Row
pyspark.sql.GroupedData
pyspark.sql.PandasCogroupedOps
pyspark.sql.DataFrameNaFunctions
pyspark.sql.DataFrameStatFunctions
pyspark.sql.Window
pyspark.sql.DataFrameReader
pyspark.sql.DataFrameWriter
pyspark.sql.DataFrameWriterV2
pyspark.sql.UDFRegistration
pyspark.sql.UDTFRegistration
pyspark.sql.udf.UserDefinedFunction
pyspark.sql.udtf.UserDefinedTableFunction
Spark Session
pyspark.sql.SparkSession.active
pyspark.sql.SparkSession.builder.appName
pyspark.sql.SparkSession.builder.config
pyspark.sql.SparkSession.builder.enableHiveSupport
pyspark.sql.SparkSession.builder.getOrCreate
pyspark.sql.SparkSession.builder.master
pyspark.sql.SparkSession.builder.remote
pyspark.sql.SparkSession.catalog
pyspark.sql.SparkSession.conf
pyspark.sql.SparkSession.createDataFrame
pyspark.sql.SparkSession.getActiveSession
pyspark.sql.SparkSession.newSession
pyspark.sql.SparkSession.range
pyspark.sql.SparkSession.read
pyspark.sql.SparkSession.readStream
pyspark.sql.SparkSession.sparkContext
pyspark.sql.SparkSession.sql
pyspark.sql.SparkSession.stop
pyspark.sql.SparkSession.streams
pyspark.sql.SparkSession.table
pyspark.sql.SparkSession.udf
pyspark.sql.SparkSession.udtf
pyspark.sql.SparkSession.version
pyspark.sql.SparkSession.builder.create
pyspark.sql.SparkSession.addArtifact
pyspark.sql.SparkSession.addArtifacts
pyspark.sql.SparkSession.copyFromLocalToFs
pyspark.sql.SparkSession.client
pyspark.sql.SparkSession.interruptAll
pyspark.sql.SparkSession.interruptTag
pyspark.sql.SparkSession.interruptOperation
pyspark.sql.SparkSession.addTag
pyspark.sql.SparkSession.removeTag
pyspark.sql.SparkSession.getTags
pyspark.sql.SparkSession.clearTags
Configuration
pyspark.sql.conf.RuntimeConfig
Input/Output
pyspark.sql.DataFrameReader.csv
pyspark.sql.DataFrameReader.format
pyspark.sql.DataFrameReader.jdbc
pyspark.sql.DataFrameReader.json
pyspark.sql.DataFrameReader.load
pyspark.sql.DataFrameReader.option
pyspark.sql.DataFrameReader.options
pyspark.sql.DataFrameReader.orc
pyspark.sql.DataFrameReader.parquet
pyspark.sql.DataFrameReader.schema
pyspark.sql.DataFrameReader.table
pyspark.sql.DataFrameReader.text
pyspark.sql.DataFrameWriter.bucketBy
pyspark.sql.DataFrameWriter.csv
pyspark.sql.DataFrameWriter.format
pyspark.sql.DataFrameWriter.insertInto
pyspark.sql.DataFrameWriter.jdbc
pyspark.sql.DataFrameWriter.json
pyspark.sql.DataFrameWriter.mode
pyspark.sql.DataFrameWriter.option
pyspark.sql.DataFrameWriter.options
pyspark.sql.DataFrameWriter.orc
pyspark.sql.DataFrameWriter.parquet
pyspark.sql.DataFrameWriter.partitionBy
pyspark.sql.DataFrameWriter.save
pyspark.sql.DataFrameWriter.saveAsTable
pyspark.sql.DataFrameWriter.sortBy
pyspark.sql.DataFrameWriter.text
pyspark.sql.DataFrameWriterV2.using
pyspark.sql.DataFrameWriterV2.option
pyspark.sql.DataFrameWriterV2.options
pyspark.sql.DataFrameWriterV2.tableProperty
pyspark.sql.DataFrameWriterV2.partitionedBy
pyspark.sql.DataFrameWriterV2.create
pyspark.sql.DataFrameWriterV2.replace
pyspark.sql.DataFrameWriterV2.createOrReplace
pyspark.sql.DataFrameWriterV2.append
pyspark.sql.DataFrameWriterV2.overwrite
pyspark.sql.DataFrameWriterV2.overwritePartitions
DataFrame
pyspark.sql.DataFrame.__getattr__
pyspark.sql.DataFrame.__getitem__
pyspark.sql.DataFrame.agg
pyspark.sql.DataFrame.alias
pyspark.sql.DataFrame.approxQuantile
pyspark.sql.DataFrame.cache
pyspark.sql.DataFrame.checkpoint
pyspark.sql.DataFrame.coalesce
pyspark.sql.DataFrame.colRegex
pyspark.sql.DataFrame.collect
pyspark.sql.DataFrame.columns
pyspark.sql.DataFrame.corr
pyspark.sql.DataFrame.count
pyspark.sql.DataFrame.cov
pyspark.sql.DataFrame.createGlobalTempView
pyspark.sql.DataFrame.createOrReplaceGlobalTempView
pyspark.sql.DataFrame.createOrReplaceTempView
pyspark.sql.DataFrame.createTempView
pyspark.sql.DataFrame.crossJoin
pyspark.sql.DataFrame.crosstab
pyspark.sql.DataFrame.cube
pyspark.sql.DataFrame.describe
pyspark.sql.DataFrame.distinct
pyspark.sql.DataFrame.drop
pyspark.sql.DataFrame.dropDuplicates
pyspark.sql.DataFrame.dropDuplicatesWithinWatermark
pyspark.sql.DataFrame.drop_duplicates
pyspark.sql.DataFrame.dropna
pyspark.sql.DataFrame.dtypes
pyspark.sql.DataFrame.exceptAll
pyspark.sql.DataFrame.explain
pyspark.sql.DataFrame.fillna
pyspark.sql.DataFrame.filter
pyspark.sql.DataFrame.first
pyspark.sql.DataFrame.foreach
pyspark.sql.DataFrame.foreachPartition
pyspark.sql.DataFrame.freqItems
pyspark.sql.DataFrame.groupBy
pyspark.sql.DataFrame.head
pyspark.sql.DataFrame.hint
pyspark.sql.DataFrame.inputFiles
pyspark.sql.DataFrame.intersect
pyspark.sql.DataFrame.intersectAll
pyspark.sql.DataFrame.isEmpty
pyspark.sql.DataFrame.isLocal
pyspark.sql.DataFrame.isStreaming
pyspark.sql.DataFrame.join
pyspark.sql.DataFrame.limit
pyspark.sql.DataFrame.localCheckpoint
pyspark.sql.DataFrame.mapInPandas
pyspark.sql.DataFrame.mapInArrow
pyspark.sql.DataFrame.melt
pyspark.sql.DataFrame.na
pyspark.sql.DataFrame.observe
pyspark.sql.DataFrame.offset
pyspark.sql.DataFrame.orderBy
pyspark.sql.DataFrame.persist
pyspark.sql.DataFrame.printSchema
pyspark.sql.DataFrame.randomSplit
pyspark.sql.DataFrame.rdd
pyspark.sql.DataFrame.registerTempTable
pyspark.sql.DataFrame.repartition
pyspark.sql.DataFrame.repartitionByRange
pyspark.sql.DataFrame.replace
pyspark.sql.DataFrame.rollup
pyspark.sql.DataFrame.sameSemantics
pyspark.sql.DataFrame.sample
pyspark.sql.DataFrame.sampleBy
pyspark.sql.DataFrame.schema
pyspark.sql.DataFrame.select
pyspark.sql.DataFrame.selectExpr
pyspark.sql.DataFrame.semanticHash
pyspark.sql.DataFrame.show
pyspark.sql.DataFrame.sort
pyspark.sql.DataFrame.sortWithinPartitions
pyspark.sql.DataFrame.sparkSession
pyspark.sql.DataFrame.stat
pyspark.sql.DataFrame.storageLevel
pyspark.sql.DataFrame.subtract
pyspark.sql.DataFrame.summary
pyspark.sql.DataFrame.tail
pyspark.sql.DataFrame.take
pyspark.sql.DataFrame.to
pyspark.sql.DataFrame.toDF
pyspark.sql.DataFrame.toJSON
pyspark.sql.DataFrame.toLocalIterator
pyspark.sql.DataFrame.toPandas
pyspark.sql.DataFrame.to_pandas_on_spark
pyspark.sql.DataFrame.transform
pyspark.sql.DataFrame.union
pyspark.sql.DataFrame.unionAll
pyspark.sql.DataFrame.unionByName
pyspark.sql.DataFrame.unpersist
pyspark.sql.DataFrame.unpivot
pyspark.sql.DataFrame.where
pyspark.sql.DataFrame.withColumn
pyspark.sql.DataFrame.withColumns
pyspark.sql.DataFrame.withColumnRenamed
pyspark.sql.DataFrame.withColumnsRenamed
pyspark.sql.DataFrame.withMetadata
pyspark.sql.DataFrame.withWatermark
pyspark.sql.DataFrame.write
pyspark.sql.DataFrame.writeStream
pyspark.sql.DataFrame.writeTo
pyspark.sql.DataFrame.pandas_api
pyspark.sql.DataFrameNaFunctions.drop
pyspark.sql.DataFrameNaFunctions.fill
pyspark.sql.DataFrameNaFunctions.replace
pyspark.sql.DataFrameStatFunctions.approxQuantile
pyspark.sql.DataFrameStatFunctions.corr
pyspark.sql.DataFrameStatFunctions.cov
pyspark.sql.DataFrameStatFunctions.crosstab
pyspark.sql.DataFrameStatFunctions.freqItems
pyspark.sql.DataFrameStatFunctions.sampleBy
Column
pyspark.sql.Column.__getattr__
pyspark.sql.Column.__getitem__
pyspark.sql.Column.alias
pyspark.sql.Column.asc
pyspark.sql.Column.asc_nulls_first
pyspark.sql.Column.asc_nulls_last
pyspark.sql.Column.astype
pyspark.sql.Column.between
pyspark.sql.Column.bitwiseAND
pyspark.sql.Column.bitwiseOR
pyspark.sql.Column.bitwiseXOR
pyspark.sql.Column.cast
pyspark.sql.Column.contains
pyspark.sql.Column.desc
pyspark.sql.Column.desc_nulls_first
pyspark.sql.Column.desc_nulls_last
pyspark.sql.Column.dropFields
pyspark.sql.Column.endswith
pyspark.sql.Column.eqNullSafe
pyspark.sql.Column.getField
pyspark.sql.Column.getItem
pyspark.sql.Column.ilike
pyspark.sql.Column.isNotNull
pyspark.sql.Column.isNull
pyspark.sql.Column.isin
pyspark.sql.Column.like
pyspark.sql.Column.name
pyspark.sql.Column.otherwise
pyspark.sql.Column.over
pyspark.sql.Column.rlike
pyspark.sql.Column.startswith
pyspark.sql.Column.substr
pyspark.sql.Column.when
pyspark.sql.Column.withField
Data Types
ArrayType
BinaryType
BooleanType
ByteType
DataType
DateType
DecimalType
DoubleType
FloatType
IntegerType
LongType
MapType
NullType
ShortType
StringType
CharType
VarcharType
StructField
StructType
TimestampType
TimestampNTZType
DayTimeIntervalType
YearMonthIntervalType
Row
pyspark.sql.Row.asDict
Functions
pyspark.sql.functions.col
pyspark.sql.functions.column
pyspark.sql.functions.lit
pyspark.sql.functions.broadcast
pyspark.sql.functions.coalesce
pyspark.sql.functions.input_file_name
pyspark.sql.functions.isnan
pyspark.sql.functions.isnull
pyspark.sql.functions.monotonically_increasing_id
pyspark.sql.functions.named_struct
pyspark.sql.functions.nanvl
pyspark.sql.functions.rand
pyspark.sql.functions.randn
pyspark.sql.functions.spark_partition_id
pyspark.sql.functions.when
pyspark.sql.functions.bitwise_not
pyspark.sql.functions.bitwiseNOT
pyspark.sql.functions.expr
pyspark.sql.functions.greatest
pyspark.sql.functions.least
pyspark.sql.functions.sqrt
pyspark.sql.functions.abs
pyspark.sql.functions.acos
pyspark.sql.functions.acosh
pyspark.sql.functions.asin
pyspark.sql.functions.asinh
pyspark.sql.functions.atan
pyspark.sql.functions.atanh
pyspark.sql.functions.atan2
pyspark.sql.functions.bin
pyspark.sql.functions.cbrt
pyspark.sql.functions.ceil
pyspark.sql.functions.ceiling
pyspark.sql.functions.conv
pyspark.sql.functions.cos
pyspark.sql.functions.cosh
pyspark.sql.functions.cot
pyspark.sql.functions.csc
pyspark.sql.functions.e
pyspark.sql.functions.exp
pyspark.sql.functions.expm1
pyspark.sql.functions.factorial
pyspark.sql.functions.floor
pyspark.sql.functions.hex
pyspark.sql.functions.unhex
pyspark.sql.functions.hypot
pyspark.sql.functions.ln
pyspark.sql.functions.log
pyspark.sql.functions.log10
pyspark.sql.functions.log1p
pyspark.sql.functions.log2
pyspark.sql.functions.negate
pyspark.sql.functions.negative
pyspark.sql.functions.pi
pyspark.sql.functions.pmod
pyspark.sql.functions.positive
pyspark.sql.functions.pow
pyspark.sql.functions.power
pyspark.sql.functions.rint
pyspark.sql.functions.round
pyspark.sql.functions.bround
pyspark.sql.functions.sec
pyspark.sql.functions.shiftleft
pyspark.sql.functions.shiftright
pyspark.sql.functions.shiftrightunsigned
pyspark.sql.functions.sign
pyspark.sql.functions.signum
pyspark.sql.functions.sin
pyspark.sql.functions.sinh
pyspark.sql.functions.tan
pyspark.sql.functions.tanh
pyspark.sql.functions.toDegrees
pyspark.sql.functions.try_add
pyspark.sql.functions.try_avg
pyspark.sql.functions.try_divide
pyspark.sql.functions.try_multiply
pyspark.sql.functions.try_subtract
pyspark.sql.functions.try_sum
pyspark.sql.functions.try_to_binary
pyspark.sql.functions.try_to_number
pyspark.sql.functions.degrees
pyspark.sql.functions.toRadians
pyspark.sql.functions.radians
pyspark.sql.functions.width_bucket
pyspark.sql.functions.add_months
pyspark.sql.functions.convert_timezone
pyspark.sql.functions.curdate
pyspark.sql.functions.current_date
pyspark.sql.functions.current_timestamp
pyspark.sql.functions.current_timezone
pyspark.sql.functions.date_add
pyspark.sql.functions.date_diff
pyspark.sql.functions.date_format
pyspark.sql.functions.date_from_unix_date
pyspark.sql.functions.date_sub
pyspark.sql.functions.date_trunc
pyspark.sql.functions.dateadd
pyspark.sql.functions.datediff
pyspark.sql.functions.day
pyspark.sql.functions.date_part
pyspark.sql.functions.datepart
pyspark.sql.functions.dayofmonth
pyspark.sql.functions.dayofweek
pyspark.sql.functions.dayofyear
pyspark.sql.functions.extract
pyspark.sql.functions.second
pyspark.sql.functions.weekofyear
pyspark.sql.functions.year
pyspark.sql.functions.quarter
pyspark.sql.functions.month
pyspark.sql.functions.last_day
pyspark.sql.functions.localtimestamp
pyspark.sql.functions.make_dt_interval
pyspark.sql.functions.make_interval
pyspark.sql.functions.make_timestamp
pyspark.sql.functions.make_timestamp_ltz
pyspark.sql.functions.make_timestamp_ntz
pyspark.sql.functions.make_ym_interval
pyspark.sql.functions.minute
pyspark.sql.functions.months_between
pyspark.sql.functions.next_day
pyspark.sql.functions.hour
pyspark.sql.functions.make_date
pyspark.sql.functions.now
pyspark.sql.functions.from_unixtime
pyspark.sql.functions.unix_timestamp
pyspark.sql.functions.to_unix_timestamp
pyspark.sql.functions.to_timestamp
pyspark.sql.functions.to_timestamp_ltz
pyspark.sql.functions.to_timestamp_ntz
pyspark.sql.functions.to_date
pyspark.sql.functions.trunc
pyspark.sql.functions.from_utc_timestamp
pyspark.sql.functions.to_utc_timestamp
pyspark.sql.functions.weekday
pyspark.sql.functions.window
pyspark.sql.functions.session_window
pyspark.sql.functions.timestamp_micros
pyspark.sql.functions.timestamp_millis
pyspark.sql.functions.timestamp_seconds
pyspark.sql.functions.try_to_timestamp
pyspark.sql.functions.unix_date
pyspark.sql.functions.unix_micros
pyspark.sql.functions.unix_millis
pyspark.sql.functions.unix_seconds
pyspark.sql.functions.window_time
pyspark.sql.functions.array
pyspark.sql.functions.array_contains
pyspark.sql.functions.arrays_overlap
pyspark.sql.functions.array_join
pyspark.sql.functions.create_map
pyspark.sql.functions.slice
pyspark.sql.functions.concat
pyspark.sql.functions.array_position
pyspark.sql.functions.element_at
pyspark.sql.functions.array_append
pyspark.sql.functions.array_size
pyspark.sql.functions.array_sort
pyspark.sql.functions.array_insert
pyspark.sql.functions.array_remove
pyspark.sql.functions.array_prepend
pyspark.sql.functions.array_distinct
pyspark.sql.functions.array_intersect
pyspark.sql.functions.array_union
pyspark.sql.functions.array_except
pyspark.sql.functions.array_compact
pyspark.sql.functions.transform
pyspark.sql.functions.exists
pyspark.sql.functions.forall
pyspark.sql.functions.filter
pyspark.sql.functions.aggregate
pyspark.sql.functions.zip_with
pyspark.sql.functions.transform_keys
pyspark.sql.functions.transform_values
pyspark.sql.functions.map_filter
pyspark.sql.functions.map_from_arrays
pyspark.sql.functions.map_zip_with
pyspark.sql.functions.explode
pyspark.sql.functions.explode_outer
pyspark.sql.functions.posexplode
pyspark.sql.functions.posexplode_outer
pyspark.sql.functions.inline
pyspark.sql.functions.inline_outer
pyspark.sql.functions.get
pyspark.sql.functions.get_json_object
pyspark.sql.functions.json_tuple
pyspark.sql.functions.from_json
pyspark.sql.functions.schema_of_json
pyspark.sql.functions.to_json
pyspark.sql.functions.json_array_length
pyspark.sql.functions.json_object_keys
pyspark.sql.functions.size
pyspark.sql.functions.cardinality
pyspark.sql.functions.struct
pyspark.sql.functions.sort_array
pyspark.sql.functions.array_max
pyspark.sql.functions.array_min
pyspark.sql.functions.shuffle
pyspark.sql.functions.reverse
pyspark.sql.functions.flatten
pyspark.sql.functions.sequence
pyspark.sql.functions.array_repeat
pyspark.sql.functions.map_contains_key
pyspark.sql.functions.map_keys
pyspark.sql.functions.map_values
pyspark.sql.functions.map_entries
pyspark.sql.functions.map_from_entries
pyspark.sql.functions.arrays_zip
pyspark.sql.functions.map_concat
pyspark.sql.functions.from_csv
pyspark.sql.functions.schema_of_csv
pyspark.sql.functions.str_to_map
pyspark.sql.functions.to_csv
pyspark.sql.functions.try_element_at
pyspark.sql.functions.years
pyspark.sql.functions.months
pyspark.sql.functions.days
pyspark.sql.functions.hours
pyspark.sql.functions.bucket
pyspark.sql.functions.any_value
pyspark.sql.functions.approxCountDistinct
pyspark.sql.functions.approx_count_distinct
pyspark.sql.functions.approx_percentile
pyspark.sql.functions.array_agg
pyspark.sql.functions.avg
pyspark.sql.functions.bit_and
pyspark.sql.functions.bit_or
pyspark.sql.functions.bit_xor
pyspark.sql.functions.bool_and
pyspark.sql.functions.bool_or
pyspark.sql.functions.collect_list
pyspark.sql.functions.collect_set
pyspark.sql.functions.corr
pyspark.sql.functions.count
pyspark.sql.functions.count_distinct
pyspark.sql.functions.countDistinct
pyspark.sql.functions.count_min_sketch
pyspark.sql.functions.count_if
pyspark.sql.functions.covar_pop
pyspark.sql.functions.covar_samp
pyspark.sql.functions.every
pyspark.sql.functions.first
pyspark.sql.functions.first_value
pyspark.sql.functions.grouping
pyspark.sql.functions.grouping_id
pyspark.sql.functions.histogram_numeric
pyspark.sql.functions.hll_sketch_agg
pyspark.sql.functions.hll_union_agg
pyspark.sql.functions.kurtosis
pyspark.sql.functions.last
pyspark.sql.functions.last_value
pyspark.sql.functions.max
pyspark.sql.functions.max_by
pyspark.sql.functions.mean
pyspark.sql.functions.median
pyspark.sql.functions.min
pyspark.sql.functions.min_by
pyspark.sql.functions.mode
pyspark.sql.functions.percentile
pyspark.sql.functions.percentile_approx
pyspark.sql.functions.product
pyspark.sql.functions.reduce
pyspark.sql.functions.regr_avgx
pyspark.sql.functions.regr_avgy
pyspark.sql.functions.regr_count
pyspark.sql.functions.regr_intercept
pyspark.sql.functions.regr_r2
pyspark.sql.functions.regr_slope
pyspark.sql.functions.regr_sxx
pyspark.sql.functions.regr_sxy
pyspark.sql.functions.regr_syy
pyspark.sql.functions.skewness
pyspark.sql.functions.some
pyspark.sql.functions.std
pyspark.sql.functions.stddev
pyspark.sql.functions.stddev_pop
pyspark.sql.functions.stddev_samp
pyspark.sql.functions.sum
pyspark.sql.functions.sum_distinct
pyspark.sql.functions.sumDistinct
pyspark.sql.functions.var_pop
pyspark.sql.functions.var_samp
pyspark.sql.functions.variance
pyspark.sql.functions.cume_dist
pyspark.sql.functions.dense_rank
pyspark.sql.functions.lag
pyspark.sql.functions.lead
pyspark.sql.functions.nth_value
pyspark.sql.functions.ntile
pyspark.sql.functions.percent_rank
pyspark.sql.functions.rank
pyspark.sql.functions.row_number
pyspark.sql.functions.asc
pyspark.sql.functions.asc_nulls_first
pyspark.sql.functions.asc_nulls_last
pyspark.sql.functions.desc
pyspark.sql.functions.desc_nulls_first
pyspark.sql.functions.desc_nulls_last
pyspark.sql.functions.ascii
pyspark.sql.functions.base64
pyspark.sql.functions.bit_length
pyspark.sql.functions.btrim
pyspark.sql.functions.char
pyspark.sql.functions.character_length
pyspark.sql.functions.char_length
pyspark.sql.functions.concat_ws
pyspark.sql.functions.contains
pyspark.sql.functions.decode
pyspark.sql.functions.elt
pyspark.sql.functions.encode
pyspark.sql.functions.endswith
pyspark.sql.functions.find_in_set
pyspark.sql.functions.format_number
pyspark.sql.functions.format_string
pyspark.sql.functions.ilike
pyspark.sql.functions.initcap
pyspark.sql.functions.instr
pyspark.sql.functions.lcase
pyspark.sql.functions.length
pyspark.sql.functions.like
pyspark.sql.functions.lower
pyspark.sql.functions.left
pyspark.sql.functions.levenshtein
pyspark.sql.functions.locate
pyspark.sql.functions.lpad
pyspark.sql.functions.ltrim
pyspark.sql.functions.mask
pyspark.sql.functions.octet_length
pyspark.sql.functions.parse_url
pyspark.sql.functions.position
pyspark.sql.functions.printf
pyspark.sql.functions.rlike
pyspark.sql.functions.regexp
pyspark.sql.functions.regexp_like
pyspark.sql.functions.regexp_count
pyspark.sql.functions.regexp_extract
pyspark.sql.functions.regexp_extract_all
pyspark.sql.functions.regexp_replace
pyspark.sql.functions.regexp_substr
pyspark.sql.functions.regexp_instr
pyspark.sql.functions.replace
pyspark.sql.functions.right
pyspark.sql.functions.ucase
pyspark.sql.functions.unbase64
pyspark.sql.functions.rpad
pyspark.sql.functions.repeat
pyspark.sql.functions.rtrim
pyspark.sql.functions.soundex
pyspark.sql.functions.split
pyspark.sql.functions.split_part
pyspark.sql.functions.startswith
pyspark.sql.functions.substr
pyspark.sql.functions.substring
pyspark.sql.functions.substring_index
pyspark.sql.functions.overlay
pyspark.sql.functions.sentences
pyspark.sql.functions.to_binary
pyspark.sql.functions.to_char
pyspark.sql.functions.to_number
pyspark.sql.functions.to_varchar
pyspark.sql.functions.translate
pyspark.sql.functions.trim
pyspark.sql.functions.upper
pyspark.sql.functions.url_decode
pyspark.sql.functions.url_encode
pyspark.sql.functions.bit_count
pyspark.sql.functions.bit_get
pyspark.sql.functions.getbit
pyspark.sql.functions.call_function
pyspark.sql.functions.call_udf
pyspark.sql.functions.pandas_udf
pyspark.sql.functions.udf
pyspark.sql.functions.udtf
pyspark.sql.functions.unwrap_udt
pyspark.sql.functions.aes_decrypt
pyspark.sql.functions.aes_encrypt
pyspark.sql.functions.bitmap_bit_position
pyspark.sql.functions.bitmap_bucket_number
pyspark.sql.functions.bitmap_construct_agg
pyspark.sql.functions.bitmap_count
pyspark.sql.functions.bitmap_or_agg
pyspark.sql.functions.current_catalog
pyspark.sql.functions.current_database
pyspark.sql.functions.current_schema
pyspark.sql.functions.current_user
pyspark.sql.functions.input_file_block_length
pyspark.sql.functions.input_file_block_start
pyspark.sql.functions.md5
pyspark.sql.functions.sha
pyspark.sql.functions.sha1
pyspark.sql.functions.sha2
pyspark.sql.functions.crc32
pyspark.sql.functions.hash
pyspark.sql.functions.xxhash64
pyspark.sql.functions.assert_true
pyspark.sql.functions.raise_error
pyspark.sql.functions.reflect
pyspark.sql.functions.hll_sketch_estimate
pyspark.sql.functions.hll_union
pyspark.sql.functions.java_method
pyspark.sql.functions.stack
pyspark.sql.functions.try_aes_decrypt
pyspark.sql.functions.typeof
pyspark.sql.functions.user
pyspark.sql.functions.version
pyspark.sql.functions.equal_null
pyspark.sql.functions.ifnull
pyspark.sql.functions.isnotnull
pyspark.sql.functions.nullif
pyspark.sql.functions.nvl
pyspark.sql.functions.nvl2
pyspark.sql.functions.xpath
pyspark.sql.functions.xpath_boolean
pyspark.sql.functions.xpath_double
pyspark.sql.functions.xpath_float
pyspark.sql.functions.xpath_int
pyspark.sql.functions.xpath_long
pyspark.sql.functions.xpath_number
pyspark.sql.functions.xpath_short
pyspark.sql.functions.xpath_string
Window
pyspark.sql.Window.currentRow
pyspark.sql.Window.orderBy
pyspark.sql.Window.partitionBy
pyspark.sql.Window.rangeBetween
pyspark.sql.Window.rowsBetween
pyspark.sql.Window.unboundedFollowing
pyspark.sql.Window.unboundedPreceding
pyspark.sql.WindowSpec.orderBy
pyspark.sql.WindowSpec.partitionBy
pyspark.sql.WindowSpec.rangeBetween
pyspark.sql.WindowSpec.rowsBetween
Grouping
pyspark.sql.GroupedData.agg
pyspark.sql.GroupedData.apply
pyspark.sql.GroupedData.applyInPandas
pyspark.sql.GroupedData.applyInPandasWithState
pyspark.sql.GroupedData.avg
pyspark.sql.GroupedData.cogroup
pyspark.sql.GroupedData.count
pyspark.sql.GroupedData.max
pyspark.sql.GroupedData.mean
pyspark.sql.GroupedData.min
pyspark.sql.GroupedData.pivot
pyspark.sql.GroupedData.sum
pyspark.sql.PandasCogroupedOps.applyInPandas
Catalog
pyspark.sql.Catalog.cacheTable
pyspark.sql.Catalog.clearCache
pyspark.sql.Catalog.createExternalTable
pyspark.sql.Catalog.createTable
pyspark.sql.Catalog.currentCatalog
pyspark.sql.Catalog.currentDatabase
pyspark.sql.Catalog.databaseExists
pyspark.sql.Catalog.dropGlobalTempView
pyspark.sql.Catalog.dropTempView
pyspark.sql.Catalog.functionExists
pyspark.sql.Catalog.getDatabase
pyspark.sql.Catalog.getFunction
pyspark.sql.Catalog.getTable
pyspark.sql.Catalog.isCached
pyspark.sql.Catalog.listCatalogs
pyspark.sql.Catalog.listColumns
pyspark.sql.Catalog.listDatabases
pyspark.sql.Catalog.listFunctions
pyspark.sql.Catalog.listTables
pyspark.sql.Catalog.recoverPartitions
pyspark.sql.Catalog.refreshByPath
pyspark.sql.Catalog.refreshTable
pyspark.sql.Catalog.registerFunction
pyspark.sql.Catalog.setCurrentCatalog
pyspark.sql.Catalog.setCurrentDatabase
pyspark.sql.Catalog.tableExists
pyspark.sql.Catalog.uncacheTable
Avro
pyspark.sql.avro.functions.from_avro
pyspark.sql.avro.functions.to_avro
Observation
pyspark.sql.Observation.get
UDF
pyspark.sql.udf.UserDefinedFunction.asNondeterministic
pyspark.sql.udf.UserDefinedFunction.returnType
pyspark.sql.UDFRegistration.register
pyspark.sql.UDFRegistration.registerJavaFunction
pyspark.sql.UDFRegistration.registerJavaUDAF
UDTF
pyspark.sql.udtf.UserDefinedTableFunction.asDeterministic
pyspark.sql.udtf.UserDefinedTableFunction.returnType
pyspark.sql.UDTFRegistration.register
Protobuf
pyspark.sql.protobuf.functions.from_protobuf
pyspark.sql.protobuf.functions.to_protobuf
Pandas API on Spark
Input/Output
pyspark.pandas.range
pyspark.pandas.read_table
pyspark.pandas.DataFrame.to_table
pyspark.pandas.read_delta
pyspark.pandas.DataFrame.to_delta
pyspark.pandas.read_parquet
pyspark.pandas.DataFrame.to_parquet
pyspark.pandas.read_orc
pyspark.pandas.DataFrame.to_orc
pyspark.pandas.read_spark_io
pyspark.pandas.DataFrame.to_spark_io
pyspark.pandas.read_csv
pyspark.pandas.DataFrame.to_csv
pyspark.pandas.read_clipboard
pyspark.pandas.DataFrame.to_clipboard
pyspark.pandas.read_excel
pyspark.pandas.DataFrame.to_excel
pyspark.pandas.read_json
pyspark.pandas.DataFrame.to_json
pyspark.pandas.read_html
pyspark.pandas.DataFrame.to_html
pyspark.pandas.read_sql_table
pyspark.pandas.read_sql_query
pyspark.pandas.read_sql
General functions
pyspark.pandas.reset_option
pyspark.pandas.get_option
pyspark.pandas.set_option
pyspark.pandas.option_context
pyspark.pandas.melt
pyspark.pandas.merge
pyspark.pandas.merge_asof
pyspark.pandas.get_dummies
pyspark.pandas.concat
pyspark.pandas.sql
pyspark.pandas.broadcast
pyspark.pandas.isna
pyspark.pandas.isnull
pyspark.pandas.notna
pyspark.pandas.notnull
pyspark.pandas.to_numeric
pyspark.pandas.to_datetime
pyspark.pandas.date_range
pyspark.pandas.to_timedelta
pyspark.pandas.timedelta_range
Series
pyspark.pandas.Series
pyspark.pandas.Series.index
pyspark.pandas.Series.dtype
pyspark.pandas.Series.dtypes
pyspark.pandas.Series.ndim
pyspark.pandas.Series.name
pyspark.pandas.Series.shape
pyspark.pandas.Series.axes
pyspark.pandas.Series.size
pyspark.pandas.Series.empty
pyspark.pandas.Series.T
pyspark.pandas.Series.hasnans
pyspark.pandas.Series.values
pyspark.pandas.Series.astype
pyspark.pandas.Series.copy
pyspark.pandas.Series.bool
pyspark.pandas.Series.at
pyspark.pandas.Series.iat
pyspark.pandas.Series.loc
pyspark.pandas.Series.iloc
pyspark.pandas.Series.keys
pyspark.pandas.Series.pop
pyspark.pandas.Series.items
pyspark.pandas.Series.iteritems
pyspark.pandas.Series.item
pyspark.pandas.Series.xs
pyspark.pandas.Series.get
pyspark.pandas.Series.add
pyspark.pandas.Series.div
pyspark.pandas.Series.mul
pyspark.pandas.Series.radd
pyspark.pandas.Series.rdiv
pyspark.pandas.Series.rmul
pyspark.pandas.Series.rsub
pyspark.pandas.Series.rtruediv
pyspark.pandas.Series.sub
pyspark.pandas.Series.truediv
pyspark.pandas.Series.pow
pyspark.pandas.Series.rpow
pyspark.pandas.Series.mod
pyspark.pandas.Series.rmod
pyspark.pandas.Series.floordiv
pyspark.pandas.Series.rfloordiv
pyspark.pandas.Series.divmod
pyspark.pandas.Series.rdivmod
pyspark.pandas.Series.combine_first
pyspark.pandas.Series.lt
pyspark.pandas.Series.gt
pyspark.pandas.Series.le
pyspark.pandas.Series.ge
pyspark.pandas.Series.ne
pyspark.pandas.Series.eq
pyspark.pandas.Series.product
pyspark.pandas.Series.dot
pyspark.pandas.Series.apply
pyspark.pandas.Series.agg
pyspark.pandas.Series.aggregate
pyspark.pandas.Series.transform
pyspark.pandas.Series.map
pyspark.pandas.Series.groupby
pyspark.pandas.Series.rolling
pyspark.pandas.Series.expanding
pyspark.pandas.Series.pipe
pyspark.pandas.Series.abs
pyspark.pandas.Series.all
pyspark.pandas.Series.any
pyspark.pandas.Series.autocorr
pyspark.pandas.Series.between
pyspark.pandas.Series.clip
pyspark.pandas.Series.corr
pyspark.pandas.Series.count
pyspark.pandas.Series.cov
pyspark.pandas.Series.cummax
pyspark.pandas.Series.cummin
pyspark.pandas.Series.cumsum
pyspark.pandas.Series.cumprod
pyspark.pandas.Series.describe
pyspark.pandas.Series.ewm
pyspark.pandas.Series.filter
pyspark.pandas.Series.kurt
pyspark.pandas.Series.mad
pyspark.pandas.Series.max
pyspark.pandas.Series.mean
pyspark.pandas.Series.min
pyspark.pandas.Series.mode
pyspark.pandas.Series.nlargest
pyspark.pandas.Series.nsmallest
pyspark.pandas.Series.pct_change
pyspark.pandas.Series.prod
pyspark.pandas.Series.nunique
pyspark.pandas.Series.is_unique
pyspark.pandas.Series.quantile
pyspark.pandas.Series.rank
pyspark.pandas.Series.sem
pyspark.pandas.Series.skew
pyspark.pandas.Series.std
pyspark.pandas.Series.sum
pyspark.pandas.Series.median
pyspark.pandas.Series.var
pyspark.pandas.Series.kurtosis
pyspark.pandas.Series.unique
pyspark.pandas.Series.value_counts
pyspark.pandas.Series.round
pyspark.pandas.Series.diff
pyspark.pandas.Series.is_monotonic
pyspark.pandas.Series.is_monotonic_increasing
pyspark.pandas.Series.is_monotonic_decreasing
pyspark.pandas.Series.align
pyspark.pandas.Series.drop
pyspark.pandas.Series.droplevel
pyspark.pandas.Series.drop_duplicates
pyspark.pandas.Series.duplicated
pyspark.pandas.Series.equals
pyspark.pandas.Series.add_prefix
pyspark.pandas.Series.add_suffix
pyspark.pandas.Series.first
pyspark.pandas.Series.head
pyspark.pandas.Series.idxmax
pyspark.pandas.Series.idxmin
pyspark.pandas.Series.isin
pyspark.pandas.Series.last
pyspark.pandas.Series.rename
pyspark.pandas.Series.rename_axis
pyspark.pandas.Series.reindex
pyspark.pandas.Series.reindex_like
pyspark.pandas.Series.reset_index
pyspark.pandas.Series.sample
pyspark.pandas.Series.searchsorted
pyspark.pandas.Series.swaplevel
pyspark.pandas.Series.swapaxes
pyspark.pandas.Series.take
pyspark.pandas.Series.tail
pyspark.pandas.Series.where
pyspark.pandas.Series.mask
pyspark.pandas.Series.truncate
pyspark.pandas.Series.backfill
pyspark.pandas.Series.bfill
pyspark.pandas.Series.isna
pyspark.pandas.Series.isnull
pyspark.pandas.Series.notna
pyspark.pandas.Series.notnull
pyspark.pandas.Series.pad
pyspark.pandas.Series.dropna
pyspark.pandas.Series.fillna
pyspark.pandas.Series.interpolate
pyspark.pandas.Series.argsort
pyspark.pandas.Series.argmin
pyspark.pandas.Series.argmax
pyspark.pandas.Series.sort_index
pyspark.pandas.Series.sort_values
pyspark.pandas.Series.unstack
pyspark.pandas.Series.explode
pyspark.pandas.Series.repeat
pyspark.pandas.Series.squeeze
pyspark.pandas.Series.factorize
pyspark.pandas.Series.append
pyspark.pandas.Series.compare
pyspark.pandas.Series.replace
pyspark.pandas.Series.update
pyspark.pandas.Series.asof
pyspark.pandas.Series.resample
pyspark.pandas.Series.shift
pyspark.pandas.Series.first_valid_index
pyspark.pandas.Series.last_valid_index
pyspark.pandas.Series.at_time
pyspark.pandas.Series.between_time
pyspark.pandas.Series.spark.column
pyspark.pandas.Series.spark.transform
pyspark.pandas.Series.spark.apply
pyspark.pandas.Series.dt.date
pyspark.pandas.Series.dt.year
pyspark.pandas.Series.dt.month
pyspark.pandas.Series.dt.day
pyspark.pandas.Series.dt.hour
pyspark.pandas.Series.dt.minute
pyspark.pandas.Series.dt.second
pyspark.pandas.Series.dt.microsecond
pyspark.pandas.Series.dt.week
pyspark.pandas.Series.dt.weekofyear
pyspark.pandas.Series.dt.dayofweek
pyspark.pandas.Series.dt.weekday
pyspark.pandas.Series.dt.dayofyear
pyspark.pandas.Series.dt.quarter
pyspark.pandas.Series.dt.is_month_start
pyspark.pandas.Series.dt.is_month_end
pyspark.pandas.Series.dt.is_quarter_start
pyspark.pandas.Series.dt.is_quarter_end
pyspark.pandas.Series.dt.is_year_start
pyspark.pandas.Series.dt.is_year_end
pyspark.pandas.Series.dt.is_leap_year
pyspark.pandas.Series.dt.daysinmonth
pyspark.pandas.Series.dt.days_in_month
pyspark.pandas.Series.dt.normalize
pyspark.pandas.Series.dt.strftime
pyspark.pandas.Series.dt.round
pyspark.pandas.Series.dt.floor
pyspark.pandas.Series.dt.ceil
pyspark.pandas.Series.dt.month_name
pyspark.pandas.Series.dt.day_name
pyspark.pandas.Series.str.capitalize
pyspark.pandas.Series.str.cat
pyspark.pandas.Series.str.center
pyspark.pandas.Series.str.contains
pyspark.pandas.Series.str.count
pyspark.pandas.Series.str.decode
pyspark.pandas.Series.str.encode
pyspark.pandas.Series.str.endswith
pyspark.pandas.Series.str.extract
pyspark.pandas.Series.str.extractall
pyspark.pandas.Series.str.find
pyspark.pandas.Series.str.findall
pyspark.pandas.Series.str.get
pyspark.pandas.Series.str.get_dummies
pyspark.pandas.Series.str.index
pyspark.pandas.Series.str.isalnum
pyspark.pandas.Series.str.isalpha
pyspark.pandas.Series.str.isdigit
pyspark.pandas.Series.str.isspace
pyspark.pandas.Series.str.islower
pyspark.pandas.Series.str.isupper
pyspark.pandas.Series.str.istitle
pyspark.pandas.Series.str.isnumeric
pyspark.pandas.Series.str.isdecimal
pyspark.pandas.Series.str.join
pyspark.pandas.Series.str.len
pyspark.pandas.Series.str.ljust
pyspark.pandas.Series.str.lower
pyspark.pandas.Series.str.lstrip
pyspark.pandas.Series.str.match
pyspark.pandas.Series.str.normalize
pyspark.pandas.Series.str.pad
pyspark.pandas.Series.str.partition
pyspark.pandas.Series.str.repeat
pyspark.pandas.Series.str.replace
pyspark.pandas.Series.str.rfind
pyspark.pandas.Series.str.rindex
pyspark.pandas.Series.str.rjust
pyspark.pandas.Series.str.rpartition
pyspark.pandas.Series.str.rsplit
pyspark.pandas.Series.str.rstrip
pyspark.pandas.Series.str.slice
pyspark.pandas.Series.str.slice_replace
pyspark.pandas.Series.str.split
pyspark.pandas.Series.str.startswith
pyspark.pandas.Series.str.strip
pyspark.pandas.Series.str.swapcase
pyspark.pandas.Series.str.title
pyspark.pandas.Series.str.translate
pyspark.pandas.Series.str.upper
pyspark.pandas.Series.str.wrap
pyspark.pandas.Series.str.zfill
pyspark.pandas.Series.cat.categories
pyspark.pandas.Series.cat.ordered
pyspark.pandas.Series.cat.codes
pyspark.pandas.Series.cat.rename_categories
pyspark.pandas.Series.cat.reorder_categories
pyspark.pandas.Series.cat.add_categories
pyspark.pandas.Series.cat.remove_categories
pyspark.pandas.Series.cat.remove_unused_categories
pyspark.pandas.Series.cat.set_categories
pyspark.pandas.Series.cat.as_ordered
pyspark.pandas.Series.cat.as_unordered
pyspark.pandas.Series.plot
pyspark.pandas.Series.plot.area
pyspark.pandas.Series.plot.bar
pyspark.pandas.Series.plot.barh
pyspark.pandas.Series.plot.box
pyspark.pandas.Series.plot.density
pyspark.pandas.Series.plot.hist
pyspark.pandas.Series.plot.line
pyspark.pandas.Series.plot.pie
pyspark.pandas.Series.plot.kde
pyspark.pandas.Series.hist
pyspark.pandas.Series.to_pandas
pyspark.pandas.Series.to_numpy
pyspark.pandas.Series.to_list
pyspark.pandas.Series.to_string
pyspark.pandas.Series.to_dict
pyspark.pandas.Series.to_clipboard
pyspark.pandas.Series.to_latex
pyspark.pandas.Series.to_markdown
pyspark.pandas.Series.to_json
pyspark.pandas.Series.to_csv
pyspark.pandas.Series.to_excel
pyspark.pandas.Series.to_frame
pyspark.pandas.Series.pandas_on_spark.transform_batch
DataFrame
pyspark.pandas.DataFrame
pyspark.pandas.DataFrame.index
pyspark.pandas.DataFrame.info
pyspark.pandas.DataFrame.columns
pyspark.pandas.DataFrame.empty
pyspark.pandas.DataFrame.dtypes
pyspark.pandas.DataFrame.shape
pyspark.pandas.DataFrame.axes
pyspark.pandas.DataFrame.ndim
pyspark.pandas.DataFrame.size
pyspark.pandas.DataFrame.select_dtypes
pyspark.pandas.DataFrame.values
pyspark.pandas.DataFrame.copy
pyspark.pandas.DataFrame.isna
pyspark.pandas.DataFrame.astype
pyspark.pandas.DataFrame.isnull
pyspark.pandas.DataFrame.notna
pyspark.pandas.DataFrame.notnull
pyspark.pandas.DataFrame.bool
pyspark.pandas.DataFrame.at
pyspark.pandas.DataFrame.iat
pyspark.pandas.DataFrame.head
pyspark.pandas.DataFrame.idxmax
pyspark.pandas.DataFrame.idxmin
pyspark.pandas.DataFrame.loc
pyspark.pandas.DataFrame.iloc
pyspark.pandas.DataFrame.insert
pyspark.pandas.DataFrame.items
pyspark.pandas.DataFrame.iteritems
pyspark.pandas.DataFrame.iterrows
pyspark.pandas.DataFrame.itertuples
pyspark.pandas.DataFrame.keys
pyspark.pandas.DataFrame.pop
pyspark.pandas.DataFrame.tail
pyspark.pandas.DataFrame.xs
pyspark.pandas.DataFrame.get
pyspark.pandas.DataFrame.where
pyspark.pandas.DataFrame.mask
pyspark.pandas.DataFrame.query
pyspark.pandas.DataFrame.add
pyspark.pandas.DataFrame.radd
pyspark.pandas.DataFrame.div
pyspark.pandas.DataFrame.rdiv
pyspark.pandas.DataFrame.truediv
pyspark.pandas.DataFrame.rtruediv
pyspark.pandas.DataFrame.mul
pyspark.pandas.DataFrame.rmul
pyspark.pandas.DataFrame.sub
pyspark.pandas.DataFrame.rsub