agent-claw: automated task changes

This commit is contained in:
daniel
2026-05-06 18:55:16 -05:00
parent 38905bb1e9
commit 732b00fb66
8494 changed files with 2018127 additions and 4 deletions

View File

@@ -0,0 +1,180 @@
import type { ILoggingConfig } from './logging-config';
import type { IDataProcessor } from './processor';
import type * as iam from '../../aws-iam';
import type * as kms from '../../aws-kms';
import type * as s3 from '../../aws-s3';
import type * as cdk from '../../core';
/**
* Possible compression options Amazon Data Firehose can use to compress data on delivery.
*/
export declare class Compression {
readonly value: string;
/**
* gzip
*/
static readonly GZIP: Compression;
/**
* Hadoop-compatible Snappy
*/
static readonly HADOOP_SNAPPY: Compression;
/**
* Snappy
*/
static readonly SNAPPY: Compression;
/**
* ZIP
*/
static readonly ZIP: Compression;
/**
* Uncompressed
*/
static readonly UNCOMPRESSED: Compression;
/**
* Creates a new Compression instance with a custom value.
*/
static of(value: string): Compression;
/**
* @param value the string value of the Compression.
*/
private constructor();
}
/**
* Options for S3 record backup of a delivery stream.
*/
export declare enum BackupMode {
/**
* All records are backed up.
*/
ALL = 0,
/**
* Only records that failed to deliver or transform are backed up.
*/
FAILED = 1
}
/**
* Logging related properties for a delivery stream destination.
*/
interface DestinationLoggingProps {
/**
* Configuration that determines whether to log errors during data transformation or delivery failures,
* and specifies the CloudWatch log group for storing error logs.
*
* @default - errors will be logged and a log group will be created for you.
*/
readonly loggingConfig?: ILoggingConfig;
}
/**
* Common properties for defining a backup, intermediary, or final S3 destination for a Amazon Data Firehose delivery stream.
*/
export interface CommonDestinationS3Props {
/**
* The length of time that Firehose buffers incoming data before delivering
* it to the S3 bucket.
*
* Minimum: Duration.seconds(0) when dynamic partitioning is disabled, Duration.seconds(60) when it is enabled
* Maximum: Duration.seconds(900)
*
* @default Duration.seconds(300)
*/
readonly bufferingInterval?: cdk.Duration;
/**
* The size of the buffer that Amazon Data Firehose uses for incoming data before
* delivering it to the S3 bucket.
*
* Minimum: Size.mebibytes(1) when record data format conversion or dynamic partitioning is disabled, Size.mebibytes(64) when it is enabled
* Maximum: Size.mebibytes(128)
*
* @default Size.mebibytes(5) when record data format conversion or dynamic partitioning is disabled, Size.mebibytes(128) when it is enabled
*/
readonly bufferingSize?: cdk.Size;
/**
* The type of compression that Amazon Data Firehose uses to compress the data
* that it delivers to the Amazon S3 bucket.
*
* The compression formats SNAPPY or ZIP cannot be specified for Amazon Redshift
* destinations because they are not supported by the Amazon Redshift COPY operation
* that reads from the S3 bucket.
*
* @default - UNCOMPRESSED
*/
readonly compression?: Compression;
/**
* The AWS KMS key used to encrypt the data that it delivers to your Amazon S3 bucket.
*
* @default - Data is not encrypted.
*/
readonly encryptionKey?: kms.IKey;
/**
* A prefix that Amazon Data Firehose evaluates and adds to failed records before writing them to S3.
*
* This prefix appears immediately following the bucket name.
* @see https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html#prefix-rules
*
* @default - See the documentation above
*/
readonly errorOutputPrefix?: string;
/**
* A prefix that Amazon Data Firehose evaluates and adds to records before writing them to S3.
*
* This prefix appears immediately following the bucket name.
* @see https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html
*
* @default - "YYYY/MM/DD/HH/"
*/
readonly dataOutputPrefix?: string;
}
/**
* Properties for defining an S3 backup destination.
*
* S3 backup is available for all destinations, regardless of whether the final destination is S3 or not.
*/
export interface DestinationS3BackupProps extends DestinationLoggingProps, CommonDestinationS3Props {
/**
* The S3 bucket that will store data and failed records.
*
* @default - If `mode` is set to `BackupMode.ALL` or `BackupMode.FAILED`, a bucket will be created for you.
*/
readonly bucket?: s3.IBucket;
/**
* Indicates the mode by which incoming records should be backed up to S3, if any.
*
* If `bucket` is provided, this will be implicitly set to `BackupMode.ALL`.
*
* @default - If `bucket` is provided, the default will be `BackupMode.ALL`. Otherwise,
* source records are not backed up to S3.
*/
readonly mode?: BackupMode;
}
/**
* Generic properties for defining a delivery stream destination.
*/
export interface CommonDestinationProps extends DestinationLoggingProps {
/**
* The IAM role associated with this destination.
*
* Assumed by Amazon Data Firehose to invoke processors and write to destinations
*
* @default - a role will be created with default permissions.
*/
readonly role?: iam.IRole;
/**
* The data transformation that should be performed on the data before writing to the destination.
*
* @default - no data transformation will occur.
* @deprecated Use `processors` instead.
*/
readonly processor?: IDataProcessor;
/**
* The data transformation that should be performed on the data before writing to the destination.
*
* @default - no data transformation will occur.
*/
readonly processors?: IDataProcessor[];
/**
* The configuration for backing up source records to S3.
*
* @default - source records will not be backed up to S3.
*/
readonly s3Backup?: DestinationS3BackupProps;
}
export {};

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.BackupMode=exports.Compression=void 0;const JSII_RTTI_SYMBOL_1=Symbol.for("jsii.rtti");class Compression{value;static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.Compression",version:"2.252.0"};static GZIP=new Compression("GZIP");static HADOOP_SNAPPY=new Compression("HADOOP_SNAPPY");static SNAPPY=new Compression("Snappy");static ZIP=new Compression("ZIP");static UNCOMPRESSED=new Compression("UNCOMPRESSED");static of(value){return new Compression(value)}constructor(value){this.value=value}}exports.Compression=Compression;var BackupMode;(function(BackupMode2){BackupMode2[BackupMode2.ALL=0]="ALL",BackupMode2[BackupMode2.FAILED=1]="FAILED"})(BackupMode||(exports.BackupMode=BackupMode={}));

View File

@@ -0,0 +1,216 @@
import type { Construct } from 'constructs';
import type { IDestination } from './destination';
import type { StreamEncryption } from './encryption';
import { DeliveryStreamGrants } from './kinesisfirehose-grants.generated';
import type { DeliveryStreamReference, IDeliveryStreamRef } from './kinesisfirehose.generated';
import type { ISource } from './source';
import * as cloudwatch from '../../aws-cloudwatch';
import * as ec2 from '../../aws-ec2';
import * as iam from '../../aws-iam';
import * as cdk from '../../core';
/**
* Represents an Amazon Data Firehose delivery stream.
*/
export interface IDeliveryStream extends cdk.IResource, iam.IGrantable, ec2.IConnectable, IDeliveryStreamRef {
/**
* The ARN of the delivery stream.
*
* @attribute
*/
readonly deliveryStreamArn: string;
/**
* The name of the delivery stream.
*
* @attribute
*/
readonly deliveryStreamName: string;
/**
* Grant the `grantee` identity permissions to perform `actions`.
*/
grant(grantee: iam.IGrantable, ...actions: string[]): iam.Grant;
/**
* Grant the `grantee` identity permissions to perform `firehose:PutRecord` and `firehose:PutRecordBatch` actions on this delivery stream.
*/
grantPutRecords(grantee: iam.IGrantable): iam.Grant;
/**
* Return the given named metric for this delivery stream.
*/
metric(metricName: string, props?: cloudwatch.MetricOptions): cloudwatch.Metric;
/**
* Metric for the number of bytes ingested successfully into the delivery stream over the specified time period after throttling.
*
* By default, this metric will be calculated as an average over a period of 5 minutes.
*/
metricIncomingBytes(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
/**
* Metric for the number of records ingested successfully into the delivery stream over the specified time period after throttling.
*
* By default, this metric will be calculated as an average over a period of 5 minutes.
*/
metricIncomingRecords(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
/**
* Metric for the number of bytes delivered to Amazon S3 for backup over the specified time period.
*
* By default, this metric will be calculated as an average over a period of 5 minutes.
*/
metricBackupToS3Bytes(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
/**
* Metric for the age (from getting into Amazon Data Firehose to now) of the oldest record in Amazon Data Firehose.
*
* Any record older than this age has been delivered to the Amazon S3 bucket for backup.
*
* By default, this metric will be calculated as an average over a period of 5 minutes.
*/
metricBackupToS3DataFreshness(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
/**
* Metric for the number of records delivered to Amazon S3 for backup over the specified time period.
*
* By default, this metric will be calculated as an average over a period of 5 minutes.
*/
metricBackupToS3Records(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
}
/**
* Base class for new and imported Amazon Data Firehose delivery streams.
*/
declare abstract class DeliveryStreamBase extends cdk.Resource implements IDeliveryStream {
abstract readonly deliveryStreamName: string;
abstract readonly deliveryStreamArn: string;
abstract readonly grantPrincipal: iam.IPrincipal;
/**
* Collection of grant methods for a DeliveryStream
*/
readonly grants: DeliveryStreamGrants;
/**
* Network connections between Amazon Data Firehose and other resources, i.e. Redshift cluster.
*/
readonly connections: ec2.Connections;
constructor(scope: Construct, id: string, props?: cdk.ResourceProps);
get deliveryStreamRef(): DeliveryStreamReference;
/**
* [disable-awslint:no-grants]
*/
grant(grantee: iam.IGrantable, ...actions: string[]): iam.Grant;
/**
*
* The use of this method is discouraged. Please use `grants.putRecords()` instead.
*
* [disable-awslint:no-grants]
*/
grantPutRecords(grantee: iam.IGrantable): iam.Grant;
metric(metricName: string, props?: cloudwatch.MetricOptions): cloudwatch.Metric;
metricIncomingBytes(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
metricIncomingRecords(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
metricBackupToS3Bytes(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
metricBackupToS3DataFreshness(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
metricBackupToS3Records(props?: cloudwatch.MetricOptions): cloudwatch.Metric;
private cannedMetric;
}
/**
* Options for server-side encryption of a delivery stream.
*/
export declare enum StreamEncryptionType {
/**
* Data in the stream is stored unencrypted.
*/
UNENCRYPTED = 0,
/**
* Data in the stream is stored encrypted by a KMS key managed by the customer.
*/
CUSTOMER_MANAGED = 1,
/**
* Data in the stream is stored encrypted by a KMS key owned by AWS and managed for use in multiple AWS accounts.
*/
AWS_OWNED = 2
}
/**
* Properties for a new delivery stream.
*/
export interface DeliveryStreamProps {
/**
* The destination that this delivery stream will deliver data to.
*/
readonly destination: IDestination;
/**
* A name for the delivery stream.
*
* @default - a name is generated by CloudFormation.
*/
readonly deliveryStreamName?: string;
/**
* The Kinesis data stream to use as a source for this delivery stream.
*
* @default - data must be written to the delivery stream via a direct put.
*/
readonly source?: ISource;
/**
* The IAM role associated with this delivery stream.
*
* Assumed by Amazon Data Firehose to read from sources and encrypt data server-side.
*
* @default - a role will be created with default permissions.
*/
readonly role?: iam.IRole;
/**
* Indicates the type of customer master key (CMK) to use for server-side encryption, if any.
*
* @default StreamEncryption.unencrypted()
*/
readonly encryption?: StreamEncryption;
}
/**
* A full specification of a delivery stream that can be used to import it fluently into the CDK application.
*/
export interface DeliveryStreamAttributes {
/**
* The ARN of the delivery stream.
*
* At least one of deliveryStreamArn and deliveryStreamName must be provided.
*
* @default - derived from `deliveryStreamName`.
*/
readonly deliveryStreamArn?: string;
/**
* The name of the delivery stream
*
* At least one of deliveryStreamName and deliveryStreamArn must be provided.
*
* @default - derived from `deliveryStreamArn`.
*/
readonly deliveryStreamName?: string;
/**
* The IAM role associated with this delivery stream.
*
* Assumed by Amazon Data Firehose to read from sources and encrypt data server-side.
*
* @default - the imported stream cannot be granted access to other resources as an `iam.IGrantable`.
*/
readonly role?: iam.IRole;
}
/**
* Create a Amazon Data Firehose delivery stream
*
* @resource AWS::KinesisFirehose::DeliveryStream
*/
export declare class DeliveryStream extends DeliveryStreamBase {
/** Uniquely identifies this class. */
static readonly PROPERTY_INJECTION_ID: string;
/**
* Import an existing delivery stream from its name.
*/
static fromDeliveryStreamName(scope: Construct, id: string, deliveryStreamName: string): IDeliveryStream;
/**
* Import an existing delivery stream from its ARN.
*/
static fromDeliveryStreamArn(scope: Construct, id: string, deliveryStreamArn: string): IDeliveryStream;
/**
* Import an existing delivery stream from its attributes.
*/
static fromDeliveryStreamAttributes(scope: Construct, id: string, attrs: DeliveryStreamAttributes): IDeliveryStream;
private readonly resource;
private _role?;
get deliveryStreamArn(): string;
get deliveryStreamName(): string;
get grantPrincipal(): iam.IPrincipal;
constructor(scope: Construct, id: string, props: DeliveryStreamProps);
}
export {};

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,35 @@
import type { Construct, IDependable } from 'constructs';
import type { CfnDeliveryStream } from './kinesisfirehose.generated';
/**
* An Amazon Data Firehose delivery stream destination configuration.
*/
export interface DestinationConfig {
/**
* S3 destination configuration properties.
*
* @default - S3 destination is not used.
*/
readonly extendedS3DestinationConfiguration?: CfnDeliveryStream.ExtendedS3DestinationConfigurationProperty;
/**
* Any resources that were created by the destination when binding it to the stack that must be deployed before the delivery stream is deployed.
*
* @default []
*/
readonly dependables?: IDependable[];
}
/**
* Options when binding a destination to a delivery stream.
*/
export interface DestinationBindOptions {
}
/**
* An Amazon Data Firehose delivery stream destination.
*/
export interface IDestination {
/**
* Binds this destination to the Amazon Data Firehose delivery stream.
*
* Implementers should use this method to bind resources to the stack and initialize values using the provided stream.
*/
bind(scope: Construct, options: DestinationBindOptions): DestinationConfig;
}

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0});

View File

@@ -0,0 +1,30 @@
import { StreamEncryptionType } from './delivery-stream';
import type { IKey } from '../../aws-kms';
/**
* Represents server-side encryption for an Amazon Firehose Delivery Stream.
*/
export declare abstract class StreamEncryption {
readonly type: StreamEncryptionType;
readonly encryptionKey?: IKey | undefined;
/**
* No server-side encryption is configured.
*/
static unencrypted(): StreamEncryption;
/**
* Configure server-side encryption using an AWS owned key.
*/
static awsOwnedKey(): StreamEncryption;
/**
* Configure server-side encryption using customer managed keys.
*
* @param encryptionKey the KMS key for the delivery stream.
*/
static customerManagedKey(encryptionKey?: IKey): StreamEncryption;
/**
* Constructor for StreamEncryption.
*
* @param type The type of server-side encryption for the Amazon Firehose delivery stream.
* @param encryptionKey Optional KMS key used for customer managed encryption.
*/
private constructor();
}

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.StreamEncryption=void 0;var jsiiDeprecationWarnings=()=>{var tmp=require("../../.warnings.jsii.js");return jsiiDeprecationWarnings=()=>tmp,tmp};const JSII_RTTI_SYMBOL_1=Symbol.for("jsii.rtti");var delivery_stream_1=()=>{var tmp=require("./delivery-stream");return delivery_stream_1=()=>tmp,tmp};class StreamEncryption{type;encryptionKey;static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.StreamEncryption",version:"2.252.0"};static unencrypted(){return new class extends StreamEncryption{}(delivery_stream_1().StreamEncryptionType.UNENCRYPTED)}static awsOwnedKey(){return new class extends StreamEncryption{}(delivery_stream_1().StreamEncryptionType.AWS_OWNED)}static customerManagedKey(encryptionKey){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_kms_IKey(encryptionKey)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,this.customerManagedKey),error}return new class extends StreamEncryption{}(delivery_stream_1().StreamEncryptionType.CUSTOMER_MANAGED,encryptionKey)}constructor(type,encryptionKey){this.type=type,this.encryptionKey=encryptionKey}}exports.StreamEncryption=StreamEncryption;

View File

@@ -0,0 +1,17 @@
export * from './delivery-stream';
export * from './source';
export * from './destination';
export * from './encryption';
export * from './processor';
export * from './processors/lambda-function-processor';
export * from './processors/decompression-processor';
export * from './processors/cloudwatch-log-processor';
export * from './processors/append-delimiter-to-record-processor';
export * from './processors/metadata-extraction-processor';
export * from './processors/record-deaggregation-processor';
export * from './common';
export * from './s3-bucket';
export * from './logging-config';
export * from './record-format';
export * from './kinesisfirehose.generated';
export * from './kinesisfirehose-grants.generated';

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,128 @@
export interface MetricWithDims<D> {
readonly namespace: string;
readonly metricName: string;
readonly statistic: string;
readonly dimensionsMap: D;
}
export declare class FirehoseMetrics {
static incomingBytesSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static incomingRecordsSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static backupToS3BytesSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static backupToS3DataFreshnessAverage(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static backupToS3RecordsSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static backupToS3SuccessSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static dataReadFromKinesisStreamBytesSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static dataReadFromKinesisStreamRecordsSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static deliveryToElasticsearchBytesSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static deliveryToElasticsearchRecordsSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static deliveryToElasticsearchSuccessSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static deliveryToRedshiftBytesSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static deliveryToRedshiftRecordsSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static deliveryToRedshiftSuccessSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static deliveryToS3BytesSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static deliveryToS3DataFreshnessAverage(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static deliveryToS3RecordsSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static deliveryToS3SuccessSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static deliveryToSplunkBytesSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static deliveryToSplunkDataAckLatencyAverage(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static deliveryToSplunkDataFreshnessAverage(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static deliveryToSplunkRecordsSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static deliveryToSplunkSuccessSum(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
static kinesisMillisBehindLatestAverage(this: void, dimensions: {
DeliveryStreamName: string;
}): MetricWithDims<{
DeliveryStreamName: string;
}>;
}

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.FirehoseMetrics=void 0;class FirehoseMetrics{static incomingBytesSum(dimensions){return{namespace:"AWS/Firehose",metricName:"IncomingBytes",dimensionsMap:dimensions,statistic:"Sum"}}static incomingRecordsSum(dimensions){return{namespace:"AWS/Firehose",metricName:"IncomingRecords",dimensionsMap:dimensions,statistic:"Sum"}}static backupToS3BytesSum(dimensions){return{namespace:"AWS/Firehose",metricName:"BackupToS3.Bytes",dimensionsMap:dimensions,statistic:"Sum"}}static backupToS3DataFreshnessAverage(dimensions){return{namespace:"AWS/Firehose",metricName:"BackupToS3.DataFreshness",dimensionsMap:dimensions,statistic:"Average"}}static backupToS3RecordsSum(dimensions){return{namespace:"AWS/Firehose",metricName:"BackupToS3.Records",dimensionsMap:dimensions,statistic:"Sum"}}static backupToS3SuccessSum(dimensions){return{namespace:"AWS/Firehose",metricName:"BackupToS3.Success",dimensionsMap:dimensions,statistic:"Sum"}}static dataReadFromKinesisStreamBytesSum(dimensions){return{namespace:"AWS/Firehose",metricName:"DataReadFromKinesisStream.Bytes",dimensionsMap:dimensions,statistic:"Sum"}}static dataReadFromKinesisStreamRecordsSum(dimensions){return{namespace:"AWS/Firehose",metricName:"DataReadFromKinesisStream.Records",dimensionsMap:dimensions,statistic:"Sum"}}static deliveryToElasticsearchBytesSum(dimensions){return{namespace:"AWS/Firehose",metricName:"DeliveryToElasticsearch.Bytes",dimensionsMap:dimensions,statistic:"Sum"}}static deliveryToElasticsearchRecordsSum(dimensions){return{namespace:"AWS/Firehose",metricName:"DeliveryToElasticsearch.Records",dimensionsMap:dimensions,statistic:"Sum"}}static deliveryToElasticsearchSuccessSum(dimensions){return{namespace:"AWS/Firehose",metricName:"DeliveryToElasticsearch.Success",dimensionsMap:dimensions,statistic:"Sum"}}static deliveryToRedshiftBytesSum(dimensions){return{namespace:"AWS/Firehose",metricName:"DeliveryToRedshift.Bytes",dimensionsMap:dimensions,statistic:"Sum"}}static deliveryToRedshiftRecordsSum(dimensions){return{namespace:"AWS/Firehose",metricName:"DeliveryToRedshift.Records",dimensionsMap:dimensions,statistic:"Sum"}}static deliveryToRedshiftSuccessSum(dimensions){return{namespace:"AWS/Firehose",metricName:"DeliveryToRedshift.Success",dimensionsMap:dimensions,statistic:"Sum"}}static deliveryToS3BytesSum(dimensions){return{namespace:"AWS/Firehose",metricName:"DeliveryToS3.Bytes",dimensionsMap:dimensions,statistic:"Sum"}}static deliveryToS3DataFreshnessAverage(dimensions){return{namespace:"AWS/Firehose",metricName:"DeliveryToS3.DataFreshness",dimensionsMap:dimensions,statistic:"Average"}}static deliveryToS3RecordsSum(dimensions){return{namespace:"AWS/Firehose",metricName:"DeliveryToS3.Records",dimensionsMap:dimensions,statistic:"Sum"}}static deliveryToS3SuccessSum(dimensions){return{namespace:"AWS/Firehose",metricName:"DeliveryToS3.Success",dimensionsMap:dimensions,statistic:"Sum"}}static deliveryToSplunkBytesSum(dimensions){return{namespace:"AWS/Firehose",metricName:"DeliveryToSplunk.Bytes",dimensionsMap:dimensions,statistic:"Sum"}}static deliveryToSplunkDataAckLatencyAverage(dimensions){return{namespace:"AWS/Firehose",metricName:"DeliveryToSplunk.DataAckLatency",dimensionsMap:dimensions,statistic:"Average"}}static deliveryToSplunkDataFreshnessAverage(dimensions){return{namespace:"AWS/Firehose",metricName:"DeliveryToSplunk.DataFreshness",dimensionsMap:dimensions,statistic:"Average"}}static deliveryToSplunkRecordsSum(dimensions){return{namespace:"AWS/Firehose",metricName:"DeliveryToSplunk.Records",dimensionsMap:dimensions,statistic:"Sum"}}static deliveryToSplunkSuccessSum(dimensions){return{namespace:"AWS/Firehose",metricName:"DeliveryToSplunk.Success",dimensionsMap:dimensions,statistic:"Sum"}}static kinesisMillisBehindLatestAverage(dimensions){return{namespace:"AWS/Firehose",metricName:"KinesisMillisBehindLatest",dimensionsMap:dimensions,statistic:"Average"}}}exports.FirehoseMetrics=FirehoseMetrics;

View File

@@ -0,0 +1,22 @@
import * as kinesisfirehose from "./kinesisfirehose.generated";
import * as iam from "../../aws-iam";
import * as cdk from "../../core/lib";
/**
* Collection of grant methods for a IDeliveryStreamRef
*/
export declare class DeliveryStreamGrants {
/**
* Creates grants for DeliveryStreamGrants
*/
static fromDeliveryStream(resource: kinesisfirehose.IDeliveryStreamRef): DeliveryStreamGrants;
protected readonly resource: kinesisfirehose.IDeliveryStreamRef;
private constructor();
/**
* Grant the given identity custom permissions
*/
actions(grantee: iam.IGrantable, actions: Array<string>, options?: cdk.PermissionsOptions): iam.Grant;
/**
* Grant the `grantee` identity permissions to perform `actions`.
*/
putRecords(grantee: iam.IGrantable): iam.Grant;
}

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.DeliveryStreamGrants=void 0;var jsiiDeprecationWarnings=()=>{var tmp=require("../../.warnings.jsii.js");return jsiiDeprecationWarnings=()=>tmp,tmp};const JSII_RTTI_SYMBOL_1=Symbol.for("jsii.rtti");var kinesisfirehose=()=>{var tmp=require("./kinesisfirehose.generated");return kinesisfirehose=()=>tmp,tmp},iam=()=>{var tmp=require("../../aws-iam");return iam=()=>tmp,tmp};class DeliveryStreamGrants{static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.DeliveryStreamGrants",version:"2.252.0"};static fromDeliveryStream(resource){try{jsiiDeprecationWarnings().aws_cdk_lib_interfaces_aws_kinesisfirehose_IDeliveryStreamRef(resource)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,this.fromDeliveryStream),error}return new DeliveryStreamGrants({resource})}resource;constructor(props){this.resource=props.resource}actions(grantee,actions,options={}){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_iam_IGrantable(grantee),jsiiDeprecationWarnings().aws_cdk_lib_PermissionsOptions(options)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,this.actions),error}return iam().Grant.addToPrincipal({actions,grantee,resourceArns:options.resourceArns??[kinesisfirehose().CfnDeliveryStream.arnForDeliveryStream(this.resource)]})}putRecords(grantee){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_iam_IGrantable(grantee)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,this.putRecords),error}const actions=["firehose:PutRecord","firehose:PutRecordBatch"];return this.actions(grantee,actions,{})}}exports.DeliveryStreamGrants=DeliveryStreamGrants;

File diff suppressed because it is too large Load Diff

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,44 @@
import type * as logs from '../../aws-logs';
/**
* Configuration interface for logging errors when data transformation or delivery fails.
*
* This interface defines whether logging is enabled and optionally allows specifying a
* CloudWatch Log Group for storing error logs.
*/
export interface ILoggingConfig {
/**
* If true, log errors when data transformation or data delivery fails.
*
* `true` when using `EnableLogging`, `false` when using `DisableLogging`.
*/
readonly logging: boolean;
/**
* The CloudWatch log group where log streams will be created to hold error logs.
*
* @default - if `logging` is set to `true`, a log group will be created for you.
*/
readonly logGroup?: logs.ILogGroup;
}
/**
* Enables logging for error logs with an optional custom CloudWatch log group.
*
* When this class is used, logging is enabled (`logging: true`) and
* you can optionally provide a CloudWatch log group for storing the error logs.
*
* If no log group is provided, a default one will be created automatically.
*/
export declare class EnableLogging implements ILoggingConfig {
readonly logGroup?: logs.ILogGroup;
readonly logging: boolean;
constructor(logGroup?: logs.ILogGroup);
}
/**
* Disables logging for error logs.
*
* When this class is used, logging is disabled (`logging: false`)
* and no CloudWatch log group can be specified.
*/
export declare class DisableLogging implements ILoggingConfig {
readonly logging: boolean;
constructor();
}

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.DisableLogging=exports.EnableLogging=void 0;var jsiiDeprecationWarnings=()=>{var tmp=require("../../.warnings.jsii.js");return jsiiDeprecationWarnings=()=>tmp,tmp};const JSII_RTTI_SYMBOL_1=Symbol.for("jsii.rtti");class EnableLogging{static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.EnableLogging",version:"2.252.0"};logGroup;logging;constructor(logGroup){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_logs_ILogGroup(logGroup)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,EnableLogging),error}this.logGroup=logGroup,this.logging=!0}}exports.EnableLogging=EnableLogging;class DisableLogging{static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.DisableLogging",version:"2.252.0"};logging;constructor(){this.logging=!1}}exports.DisableLogging=DisableLogging;

View File

@@ -0,0 +1,56 @@
import type { Construct, IDependable } from 'constructs';
import type * as iam from '../../../aws-iam';
import type * as kms from '../../../aws-kms';
import * as cdk from '../../../core';
import type { CommonDestinationProps, DestinationS3BackupProps } from '../common';
import type { CfnDeliveryStream } from '../kinesisfirehose.generated';
import type { ILoggingConfig } from '../logging-config';
import type { DataProcessorBindOptions } from '../processor';
import type { DynamicPartitioningProps } from '../s3-bucket';
export declare const PARTITION_KEY_QUERY = "partitionKeyFromQuery";
export declare const PARTITION_KEY_LAMBDA = "partitionKeyFromLambda";
export declare const ERROR_OUTPUT_TYPE = "!{firehose:error-output-type}";
export interface DestinationLoggingProps {
/**
* Configuration that determines whether to log errors during data transformation or delivery failures,
* and specifies the CloudWatch log group for storing error logs.
*
* @default - errors will be logged and a log group will be created for you.
*/
readonly loggingConfig?: ILoggingConfig;
/**
* The IAM role associated with this destination.
*/
readonly role: iam.IRole;
/**
* The ID of the stream that is created in the log group where logs will be placed.
*
* Must be unique within the log group, so should be different every time this function is called.
*/
readonly streamId: string;
}
interface ConfigWithDependables {
/**
* Resources that were created by the sub-config creator that must be deployed before the delivery stream is deployed.
*/
readonly dependables: IDependable[];
}
export interface DestinationLoggingConfig extends ConfigWithDependables {
/**
* Logging options that will be injected into the destination configuration.
*/
readonly loggingOptions: CfnDeliveryStream.CloudWatchLoggingOptionsProperty;
}
export interface DestinationBackupConfig extends ConfigWithDependables {
/**
* S3 backup configuration that will be injected into the destination configuration.
*/
readonly backupConfig: CfnDeliveryStream.S3DestinationConfigurationProperty;
}
export declare function createLoggingOptions(scope: Construct, props: DestinationLoggingProps): DestinationLoggingConfig | undefined;
export declare function createBufferingHints(scope: Construct, interval?: cdk.Duration, size?: cdk.Size, dataFormatConversionEnabled?: boolean, dynamicPartitioningEnabled?: boolean): CfnDeliveryStream.BufferingHintsProperty | undefined;
export declare function createEncryptionConfig(role: iam.IRole, encryptionKey?: kms.IKey): CfnDeliveryStream.EncryptionConfigurationProperty | undefined;
export declare function createProcessingConfig(scope: Construct, props: CommonDestinationProps, options: DataProcessorBindOptions): CfnDeliveryStream.ProcessingConfigurationProperty | undefined;
export declare function createBackupConfig(scope: Construct, role: iam.IRole, props?: DestinationS3BackupProps): DestinationBackupConfig | undefined;
export declare function createDynamicPartitioningConfiguration(scope: Construct, props?: DynamicPartitioningProps): CfnDeliveryStream.DynamicPartitioningConfigurationProperty | undefined;
export {};

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,100 @@
import type { Construct } from 'constructs';
import type { CfnDeliveryStream } from './kinesisfirehose.generated';
import type * as iam from '../../aws-iam';
import type { Duration, Size } from '../../core';
/**
* Configure the LambdaFunctionProcessor.
*/
export interface DataProcessorProps {
/**
* The length of time Amazon Data Firehose will buffer incoming data before calling the processor.
*s
* @default Duration.minutes(1)
*/
readonly bufferInterval?: Duration;
/**
* The amount of incoming data Amazon Data Firehose will buffer before calling the processor.
*
* @default Size.mebibytes(3)
*/
readonly bufferSize?: Size;
/**
* The number of times Amazon Data Firehose will retry the processor invocation after a failure due to network timeout or invocation limits.
*
* @default 3
*/
readonly retries?: number;
}
/**
* The key-value pair that identifies the underlying processor resource.
*
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-processorparameter.html
*/
export interface DataProcessorIdentifier {
/**
* The parameter name that corresponds to the processor resource's identifier.
*/
readonly parameterName: string;
/**
* The identifier of the underlying processor resource.
*/
readonly parameterValue: string;
}
/**
* The full configuration of a data processor.
*
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-kinesisfirehose-deliverystream-processor.html
*/
export interface DataProcessorConfig {
/**
* The type of processor.
*/
readonly processorType: string;
/**
* The key-value pair that identifies the underlying processor resource.
*
* Ignored when the `parameters` is specified.
*/
readonly processorIdentifier: DataProcessorIdentifier;
/**
* The processor parameters.
*
* @default - No processor parameters
*/
readonly parameters?: CfnDeliveryStream.ProcessorParameterProperty[];
}
/**
* Options when binding a DataProcessor to a delivery stream destination.
*/
export interface DataProcessorBindOptions {
/**
* The IAM role assumed by Amazon Data Firehose to write to the destination that this DataProcessor will bind to.
*/
readonly role: iam.IRole;
/**
* Whether the dynamic partitioning is enabled.
* @default false
*/
readonly dynamicPartitioningEnabled?: boolean;
/**
* S3 bucket prefix
* @default - No prefix
*/
readonly prefix?: string;
}
/**
* A data processor that Amazon Data Firehose will call to transform records before delivering data.
*/
export interface IDataProcessor {
/**
* The constructor props of the DataProcessor.
*/
readonly props: DataProcessorProps;
/**
* Binds this processor to a destination of a delivery stream.
*
* Implementers should use this method to grant processor invocation permissions to the provided stream and return the
* necessary configuration to register as a processor.
*/
bind(scope: Construct, options: DataProcessorBindOptions): DataProcessorConfig;
}

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0});

View File

@@ -0,0 +1,12 @@
import type { Construct } from 'constructs';
import type { DataProcessorBindOptions, DataProcessorConfig, DataProcessorProps, IDataProcessor } from '../processor';
/**
* The data processor to append new line delimiter to each record.
*
* @see https://docs.aws.amazon.com/firehose/latest/dev/dynamic-partitioning-s3bucketprefix.html#dynamic-partitioning-new-line-delimiter
*/
export declare class AppendDelimiterToRecordProcessor implements IDataProcessor {
readonly props: DataProcessorProps;
constructor();
bind(_scope: Construct, _options: DataProcessorBindOptions): DataProcessorConfig;
}

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.AppendDelimiterToRecordProcessor=void 0;var jsiiDeprecationWarnings=()=>{var tmp=require("../../../.warnings.jsii.js");return jsiiDeprecationWarnings=()=>tmp,tmp};const JSII_RTTI_SYMBOL_1=Symbol.for("jsii.rtti");class AppendDelimiterToRecordProcessor{static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.AppendDelimiterToRecordProcessor",version:"2.252.0"};props={};constructor(){}bind(_scope,_options){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesisfirehose_DataProcessorBindOptions(_options)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,this.bind),error}return{processorType:"AppendDelimiterToRecord",processorIdentifier:{parameterName:"",parameterValue:""},parameters:[]}}}exports.AppendDelimiterToRecordProcessor=AppendDelimiterToRecordProcessor;

View File

@@ -0,0 +1,23 @@
import type { Construct } from 'constructs';
import type { DataProcessorBindOptions, DataProcessorConfig, DataProcessorProps, IDataProcessor } from '../processor';
/**
* Options for CloudWatchLogProcessor.
*/
export interface CloudWatchLogProcessorOptions {
/**
* Extract message from CloudWatch logs.
* This must be true.
*/
readonly dataMessageExtraction: boolean;
}
/**
* The data processor to extract message after decompression of CloudWatch Logs.
* This processor must used with `DecompressionProcessor`
*
* @see https://docs.aws.amazon.com/firehose/latest/dev/Message_extraction.html
*/
export declare class CloudWatchLogProcessor implements IDataProcessor {
readonly props: DataProcessorProps;
constructor(options: CloudWatchLogProcessorOptions);
bind(_scope: Construct, _options: DataProcessorBindOptions): DataProcessorConfig;
}

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.CloudWatchLogProcessor=void 0;var jsiiDeprecationWarnings=()=>{var tmp=require("../../../.warnings.jsii.js");return jsiiDeprecationWarnings=()=>tmp,tmp};const JSII_RTTI_SYMBOL_1=Symbol.for("jsii.rtti");var core_1=()=>{var tmp=require("../../../core");return core_1=()=>tmp,tmp},literal_string_1=()=>{var tmp=require("../../../core/lib/private/literal-string");return literal_string_1=()=>tmp,tmp};class CloudWatchLogProcessor{static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.CloudWatchLogProcessor",version:"2.252.0"};props={};constructor(options){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesisfirehose_CloudWatchLogProcessorOptions(options)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,CloudWatchLogProcessor),error}if(!options.dataMessageExtraction)throw new(core_1()).UnscopedValidationError((0,literal_string_1().lit)`DataMessageExtractionMustBeTrue`,"dataMessageExtraction must be true.")}bind(_scope,_options){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesisfirehose_DataProcessorBindOptions(_options)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,this.bind),error}return{processorType:"CloudWatchLogProcessing",processorIdentifier:{parameterName:"",parameterValue:""},parameters:[{parameterName:"DataMessageExtraction",parameterValue:"true"}]}}}exports.CloudWatchLogProcessor=CloudWatchLogProcessor;

View File

@@ -0,0 +1,41 @@
import type { Construct } from 'constructs';
import type { DataProcessorBindOptions, DataProcessorConfig, DataProcessorProps, IDataProcessor } from '../processor';
/**
* Compression format for DecompressionProcessor.
*/
export declare class DecompressionProcessorCompressionFormat {
readonly compressionFormat: string;
/**
* GZIP compression
*/
static readonly GZIP: DecompressionProcessorCompressionFormat;
/**
* A custom compression format
*/
static of(compressionFormat: string): DecompressionProcessorCompressionFormat;
/**
* @param compressionFormat The compression format string
*/
private constructor();
}
/**
* Options for DecompressionProcessor.
*/
export interface DecompressionProcessorOptions {
/**
* The input compression format
* @default DecompressionProcessorCompressionFormat.GZIP
*/
readonly compressionFormat?: DecompressionProcessorCompressionFormat;
}
/**
* The data processor to decompress CloudWatch Logs.
*
* @see https://docs.aws.amazon.com/firehose/latest/dev/writing-with-cloudwatch-logs-decompression.html
*/
export declare class DecompressionProcessor implements IDataProcessor {
private readonly options;
readonly props: DataProcessorProps;
constructor(options?: DecompressionProcessorOptions);
bind(_scope: Construct, _options: DataProcessorBindOptions): DataProcessorConfig;
}

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.DecompressionProcessor=exports.DecompressionProcessorCompressionFormat=void 0;var jsiiDeprecationWarnings=()=>{var tmp=require("../../../.warnings.jsii.js");return jsiiDeprecationWarnings=()=>tmp,tmp};const JSII_RTTI_SYMBOL_1=Symbol.for("jsii.rtti");class DecompressionProcessorCompressionFormat{compressionFormat;static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.DecompressionProcessorCompressionFormat",version:"2.252.0"};static GZIP=DecompressionProcessorCompressionFormat.of("GZIP");static of(compressionFormat){return new DecompressionProcessorCompressionFormat(compressionFormat)}constructor(compressionFormat){this.compressionFormat=compressionFormat}}exports.DecompressionProcessorCompressionFormat=DecompressionProcessorCompressionFormat;class DecompressionProcessor{options;static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.DecompressionProcessor",version:"2.252.0"};props={};constructor(options={}){this.options=options;try{jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesisfirehose_DecompressionProcessorOptions(options)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,DecompressionProcessor),error}}bind(_scope,_options){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesisfirehose_DataProcessorBindOptions(_options)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,this.bind),error}return{processorType:"Decompression",processorIdentifier:{parameterName:"",parameterValue:""},parameters:[{parameterName:"CompressionFormat",parameterValue:this.options.compressionFormat?.compressionFormat??"GZIP"}]}}}exports.DecompressionProcessor=DecompressionProcessor;

View File

@@ -0,0 +1,15 @@
import type { Construct } from 'constructs';
import type * as lambda from '../../../aws-lambda';
import type { DataProcessorBindOptions, DataProcessorConfig, DataProcessorProps, IDataProcessor } from '../processor';
/**
* Use an AWS Lambda function to transform records.
*/
export declare class LambdaFunctionProcessor implements IDataProcessor {
private readonly lambdaFunction;
/**
* The constructor props of the LambdaFunctionProcessor.
*/
readonly props: DataProcessorProps;
constructor(lambdaFunction: lambda.IFunction, props?: DataProcessorProps);
bind(_scope: Construct, options: DataProcessorBindOptions): DataProcessorConfig;
}

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.LambdaFunctionProcessor=void 0;var jsiiDeprecationWarnings=()=>{var tmp=require("../../../.warnings.jsii.js");return jsiiDeprecationWarnings=()=>tmp,tmp};const JSII_RTTI_SYMBOL_1=Symbol.for("jsii.rtti");class LambdaFunctionProcessor{lambdaFunction;static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.LambdaFunctionProcessor",version:"2.252.0"};props;constructor(lambdaFunction,props={}){this.lambdaFunction=lambdaFunction;try{jsiiDeprecationWarnings().aws_cdk_lib_aws_lambda_IFunction(lambdaFunction),jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesisfirehose_DataProcessorProps(props)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,LambdaFunctionProcessor),error}this.props=props}bind(_scope,options){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesisfirehose_DataProcessorBindOptions(options)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,this.bind),error}return this.lambdaFunction.grantInvoke(options.role),{processorType:"Lambda",processorIdentifier:{parameterName:"LambdaArn",parameterValue:this.lambdaFunction.functionArn}}}}exports.LambdaFunctionProcessor=LambdaFunctionProcessor;

View File

@@ -0,0 +1,51 @@
import type { Construct } from 'constructs';
import type { DataProcessorBindOptions, DataProcessorConfig, DataProcessorProps, IDataProcessor } from '../processor';
/**
* Props for MetadataExtractionProcessor
*/
export interface MetadataExtractionProcessorOptions {
/**
* Map parameter to JQ query
*/
readonly metadataExtractionQuery: string;
/**
* JSON parsing engine
*/
readonly jsonParsingEngine: JsonParsingEngine;
}
/**
* The JSON parsing engine for MetadataExtractionProcessor
*/
export declare class JsonParsingEngine {
readonly parsingEngine: string;
/**
* The JQ 1.6 parsing engine
*/
static readonly JQ_1_6: JsonParsingEngine;
/**
* A custom parsing engine
*/
static of(parsingEngine: string): JsonParsingEngine;
/**
* @param parsingEngine The parsing engine string
*/
private constructor();
}
/**
* The data processor for dynamic partitioning with inline parsing.
*
* @see https://docs.aws.amazon.com/firehose/latest/dev/dynamic-partitioning-partitioning-keys.html
*/
export declare class MetadataExtractionProcessor implements IDataProcessor {
private readonly options;
private readonly keys;
/**
* Creates the inline parsing configuration with JQ 1.6 engine.
*
* @param query A map of partition key to jq expression.
*/
static jq16(query: Record<string, string>): MetadataExtractionProcessor;
readonly props: DataProcessorProps;
constructor(options: MetadataExtractionProcessorOptions, keys: string[]);
bind(scope: Construct, options: DataProcessorBindOptions): DataProcessorConfig;
}

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.MetadataExtractionProcessor=exports.JsonParsingEngine=void 0;var jsiiDeprecationWarnings=()=>{var tmp=require("../../../.warnings.jsii.js");return jsiiDeprecationWarnings=()=>tmp,tmp};const JSII_RTTI_SYMBOL_1=Symbol.for("jsii.rtti");var core_1=()=>{var tmp=require("../../../core");return core_1=()=>tmp,tmp},literal_string_1=()=>{var tmp=require("../../../core/lib/private/literal-string");return literal_string_1=()=>tmp,tmp};class JsonParsingEngine{parsingEngine;static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.JsonParsingEngine",version:"2.252.0"};static JQ_1_6=JsonParsingEngine.of("JQ-1.6");static of(parsingEngine){return new JsonParsingEngine(parsingEngine)}constructor(parsingEngine){this.parsingEngine=parsingEngine}}exports.JsonParsingEngine=JsonParsingEngine;class MetadataExtractionProcessor{options;keys;static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.MetadataExtractionProcessor",version:"2.252.0"};static jq16(query){const keys=Object.keys(query);if(keys.length===0)throw new(core_1()).UnscopedValidationError((0,literal_string_1().lit)`MetadataExtractionQueryCannotBeEmpty`,"The query for MetadataExtractionProcessor should not be empty.");if(keys.length>50)throw new(core_1()).UnscopedValidationError((0,literal_string_1().lit)`MetadataExtractionQueryExceedsLimit`,"The query for MetadataExtractionProcessor cannot exceed the limit of 50 keys.");const jqQuery=keys.map(key=>`${JSON.stringify(key)}:${query[key]}`),options={jsonParsingEngine:JsonParsingEngine.JQ_1_6,metadataExtractionQuery:`{${jqQuery.join(",")}}`};return new MetadataExtractionProcessor(options,keys)}props={};constructor(options,keys){this.options=options,this.keys=keys;try{jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesisfirehose_MetadataExtractionProcessorOptions(options)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,MetadataExtractionProcessor),error}}bind(scope,options){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesisfirehose_DataProcessorBindOptions(options)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,this.bind),error}if(!options.dynamicPartitioningEnabled)throw new(core_1()).ValidationError((0,literal_string_1().lit)`MetadataExtractionProcessorRequiresDynamicPartitioning`,"MetadataExtractionProcessor can only be present when Dynamic Partitioning is enabled.",scope);const re=/!\{partitionKeyFromQuery:([^{}]+)\}/g,usedKeys=new Set;let match;for(;match=re.exec(options.prefix??"");)usedKeys.add(match[1]);if(!(this.keys.length===usedKeys.size&&this.keys.every(key=>usedKeys.has(key))))throw new(core_1()).ValidationError((0,literal_string_1().lit)`DynamicPartitioningInlineParsingKeyMismatch`,"When dynamic partitioning via inline parsing is enabled, you must use all specified dynamic partitioning key values for partitioning your data source.",scope);const parameters=[{parameterName:"MetadataExtractionQuery",parameterValue:this.options.metadataExtractionQuery},{parameterName:"JsonParsingEngine",parameterValue:this.options.jsonParsingEngine.parsingEngine}];return{processorType:"MetadataExtraction",processorIdentifier:{parameterName:"",parameterValue:""},parameters}}}exports.MetadataExtractionProcessor=MetadataExtractionProcessor;

View File

@@ -0,0 +1,48 @@
import type { Construct } from 'constructs';
import type { DataProcessorBindOptions, DataProcessorConfig, DataProcessorProps, IDataProcessor } from '../processor';
/**
* Props for RecordDeAggregationProcessor
*/
export interface RecordDeAggregationProcessorOptions {
/**
* The sub-record type to deaggregate input records.
*/
readonly subRecordType: SubRecordType;
/**
* The custom delimiter when subRecordType is DELIMITED. Must be specified in the base64-encoded format.
* @default - No delimiter
*/
readonly delimiter?: string;
}
/**
* The sub-record type to deaggregate input records.
*/
export declare enum SubRecordType {
/** The records are JSON objects on a single line with no delimiter or newline-delimited (JSONL). */
JSON = "JSON",
/** The records are delimited by a custom delimiter. */
DELIMITED = "DELIMITED"
}
/**
* The data processor for multi record deaggrecation.
*
* Record deaggregation by JSON or by delimiter is capped at 500 per record.
*
* @see https://docs.aws.amazon.com/firehose/latest/dev/dynamic-partitioning-multirecord-deaggergation.html
*/
export declare class RecordDeAggregationProcessor implements IDataProcessor {
private readonly options;
/**
* Perform deaggregation from JSON objects on a single line with no delimiter or newline-delimited (JSONL).
*/
static json(): RecordDeAggregationProcessor;
/**
* Perform deaggregation based on a specified custom delimiter.
*
* @param delimiter The custom delimiter.
*/
static delimited(delimiter: string): RecordDeAggregationProcessor;
readonly props: DataProcessorProps;
constructor(options: RecordDeAggregationProcessorOptions);
bind(scope: Construct, options: DataProcessorBindOptions): DataProcessorConfig;
}

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.RecordDeAggregationProcessor=exports.SubRecordType=void 0;var jsiiDeprecationWarnings=()=>{var tmp=require("../../../.warnings.jsii.js");return jsiiDeprecationWarnings=()=>tmp,tmp};const JSII_RTTI_SYMBOL_1=Symbol.for("jsii.rtti");var core_1=()=>{var tmp=require("../../../core");return core_1=()=>tmp,tmp},literal_string_1=()=>{var tmp=require("../../../core/lib/private/literal-string");return literal_string_1=()=>tmp,tmp},SubRecordType;(function(SubRecordType2){SubRecordType2.JSON="JSON",SubRecordType2.DELIMITED="DELIMITED"})(SubRecordType||(exports.SubRecordType=SubRecordType={}));class RecordDeAggregationProcessor{options;static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.RecordDeAggregationProcessor",version:"2.252.0"};static json(){return new RecordDeAggregationProcessor({subRecordType:SubRecordType.JSON})}static delimited(delimiter){return new RecordDeAggregationProcessor({subRecordType:SubRecordType.DELIMITED,delimiter:Buffer.from(delimiter).toString("base64")})}props={};constructor(options){this.options=options;try{jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesisfirehose_RecordDeAggregationProcessorOptions(options)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,RecordDeAggregationProcessor),error}}bind(scope,options){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesisfirehose_DataProcessorBindOptions(options)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,this.bind),error}const parameters=[{parameterName:"SubRecordType",parameterValue:this.options.subRecordType}];if(!options.dynamicPartitioningEnabled)throw new(core_1()).ValidationError((0,literal_string_1().lit)`RecordDeAggregationProcessorRequiresDynamicPartitioning`,"RecordDeAggregationProcessor can only be present when Dynamic Partitioning is enabled.",scope);if(this.options.subRecordType===SubRecordType.DELIMITED){if(!this.options.delimiter)throw new(core_1()).ValidationError((0,literal_string_1().lit)`DelimiterRequiredForDelimitedSubRecordType`,"The delimiter must be specified when subRecordType is DELIMITED.",scope);parameters.push({parameterName:"Delimiter",parameterValue:this.options.delimiter})}return{processorType:"RecordDeAggregation",processorIdentifier:{parameterName:"",parameterValue:""},parameters}}}exports.RecordDeAggregationProcessor=RecordDeAggregationProcessor;

View File

@@ -0,0 +1,3 @@
export * from './input';
export * from './output';
export * from './schema';

View File

@@ -0,0 +1 @@
"use strict";var __createBinding=exports&&exports.__createBinding||(Object.create?(function(o,m,k,k2){k2===void 0&&(k2=k);var desc=Object.getOwnPropertyDescriptor(m,k);(!desc||("get"in desc?!m.__esModule:desc.writable||desc.configurable))&&(desc={enumerable:!0,get:function(){return m[k]}}),Object.defineProperty(o,k2,desc)}):(function(o,m,k,k2){k2===void 0&&(k2=k),o[k2]=m[k]})),__exportStar=exports&&exports.__exportStar||function(m,exports2){for(var p in m)p!=="default"&&!Object.prototype.hasOwnProperty.call(exports2,p)&&__createBinding(exports2,m,p)};Object.defineProperty(exports,"__esModule",{value:!0});var _noFold;exports.OpenXJsonInputFormat=void 0,Object.defineProperty(exports,_noFold="OpenXJsonInputFormat",{enumerable:!0,configurable:!0,get:()=>{var value=require("./input").OpenXJsonInputFormat;return Object.defineProperty(exports,_noFold="OpenXJsonInputFormat",{enumerable:!0,configurable:!0,value}),value}}),exports.TimestampParser=void 0,Object.defineProperty(exports,_noFold="TimestampParser",{enumerable:!0,configurable:!0,get:()=>{var value=require("./input").TimestampParser;return Object.defineProperty(exports,_noFold="TimestampParser",{enumerable:!0,configurable:!0,value}),value}}),exports.HiveJsonInputFormat=void 0,Object.defineProperty(exports,_noFold="HiveJsonInputFormat",{enumerable:!0,configurable:!0,get:()=>{var value=require("./input").HiveJsonInputFormat;return Object.defineProperty(exports,_noFold="HiveJsonInputFormat",{enumerable:!0,configurable:!0,value}),value}}),exports.InputFormat=void 0,Object.defineProperty(exports,_noFold="InputFormat",{enumerable:!0,configurable:!0,get:()=>{var value=require("./input").InputFormat;return Object.defineProperty(exports,_noFold="InputFormat",{enumerable:!0,configurable:!0,value}),value}}),exports.ParquetCompression=void 0,Object.defineProperty(exports,_noFold="ParquetCompression",{enumerable:!0,configurable:!0,get:()=>{var value=require("./output").ParquetCompression;return Object.defineProperty(exports,_noFold="ParquetCompression",{enumerable:!0,configurable:!0,value}),value}}),exports.ParquetWriterVersion=void 0,Object.defineProperty(exports,_noFold="ParquetWriterVersion",{enumerable:!0,configurable:!0,get:()=>{var value=require("./output").ParquetWriterVersion;return Object.defineProperty(exports,_noFold="ParquetWriterVersion",{enumerable:!0,configurable:!0,value}),value}}),exports.ParquetOutputFormat=void 0,Object.defineProperty(exports,_noFold="ParquetOutputFormat",{enumerable:!0,configurable:!0,get:()=>{var value=require("./output").ParquetOutputFormat;return Object.defineProperty(exports,_noFold="ParquetOutputFormat",{enumerable:!0,configurable:!0,value}),value}}),exports.OrcCompression=void 0,Object.defineProperty(exports,_noFold="OrcCompression",{enumerable:!0,configurable:!0,get:()=>{var value=require("./output").OrcCompression;return Object.defineProperty(exports,_noFold="OrcCompression",{enumerable:!0,configurable:!0,value}),value}}),exports.OrcFormatVersion=void 0,Object.defineProperty(exports,_noFold="OrcFormatVersion",{enumerable:!0,configurable:!0,get:()=>{var value=require("./output").OrcFormatVersion;return Object.defineProperty(exports,_noFold="OrcFormatVersion",{enumerable:!0,configurable:!0,value}),value}}),exports.OrcOutputFormat=void 0,Object.defineProperty(exports,_noFold="OrcOutputFormat",{enumerable:!0,configurable:!0,get:()=>{var value=require("./output").OrcOutputFormat;return Object.defineProperty(exports,_noFold="OrcOutputFormat",{enumerable:!0,configurable:!0,value}),value}}),exports.OutputFormat=void 0,Object.defineProperty(exports,_noFold="OutputFormat",{enumerable:!0,configurable:!0,get:()=>{var value=require("./output").OutputFormat;return Object.defineProperty(exports,_noFold="OutputFormat",{enumerable:!0,configurable:!0,value}),value}}),exports.SchemaConfiguration=void 0,Object.defineProperty(exports,_noFold="SchemaConfiguration",{enumerable:!0,configurable:!0,get:()=>{var value=require("./schema").SchemaConfiguration;return Object.defineProperty(exports,_noFold="SchemaConfiguration",{enumerable:!0,configurable:!0,value}),value}});

View File

@@ -0,0 +1,119 @@
import type { CfnDeliveryStream } from '../kinesisfirehose.generated';
/**
* An input format to be used in Firehose record format conversion.
*/
export interface IInputFormat {
/**
* Renders the cloudformation properties for the input format.
*/
createInputFormatConfig(): CfnDeliveryStream.InputFormatConfigurationProperty;
}
/**
* Props for OpenX JSON input format for data record format conversion
*/
export interface OpenXJsonInputFormatProps {
/**
* Whether the JSON keys should be lowercased when written as column names
*
* @default `true`
*/
readonly lowercaseColumnNames?: boolean;
/**
* Maps column names to JSON keys that aren't identical to the column names.
* This is useful when the JSON contains keys that are Hive keywords.
* For example, `timestamp` is a Hive keyword. If you have a JSON key named `timestamp`, set this parameter to `{"ts": "timestamp"}` to map this key to a column named `ts`
*
* @default JSON keys are not renamed
*/
readonly columnToJsonKeyMappings?: Record<string, string>;
/**
* When set to `true`, specifies that the names of the keys include dots and that you want Firehose to replace them with underscores.
* This is useful because Apache Hive does not allow dots in column names.
* For example, if the JSON contains a key whose name is "a.b", you can define the column name to be "a_b" when using this option.
*
* @default `false`
*/
readonly convertDotsInJsonKeysToUnderscores?: boolean;
}
/**
* This class specifies properties for OpenX JSON input format for record format conversion.
*
* You should only need to specify an instance of this class if the default configuration does not suit your needs.
*/
export declare class OpenXJsonInputFormat implements IInputFormat {
/**
* Properties for OpenX JSON input format
*/
readonly props?: OpenXJsonInputFormatProps;
constructor(props?: OpenXJsonInputFormatProps);
private createOpenXJsonSerde;
createInputFormatConfig(): CfnDeliveryStream.InputFormatConfigurationProperty;
}
/**
* Value class that wraps a Joda Time format string.
* Use this with the Hive JSON input format for data record format conversion to parse custom timestamp formats.
*/
export declare class TimestampParser {
/**
* Parses timestamps formatted in milliseconds since epoch.
*/
static readonly EPOCH_MILLIS: TimestampParser;
/**
* Creates a TimestampParser from the given format string.
*
* The format string should be a valid Joda Time pattern string.
* See [Class DateTimeFormat](https://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html) for more details
*
* @param format the Joda Time format string
*/
static fromFormatString(format: string): TimestampParser;
/**
* The format string to use in Hive JSON input format configuration.
*/
readonly format: string;
private constructor();
}
/**
* Props for Hive JSON input format for data record format conversion
*/
export interface HiveJsonInputFormatProps {
/**
* List of TimestampParsers.
*
* These are used to parse custom timestamp strings from input JSON into dates.
*
* Note: Specifying a parser will override the default timestamp parser. If the default timestamp parser is required,
* include `TimestampParser.DEFAULT` in the list of parsers along with the custom parser.
*
* @default the default timestamp parser is used
*/
readonly timestampParsers?: TimestampParser[];
}
/**
* This class specifies properties for Hive JSON input format for record format conversion.
*
* You should only need to specify an instance of this class if the default configuration does not suit your needs.
*/
export declare class HiveJsonInputFormat implements IInputFormat {
/**
* Properties for Hive JSON input format
*/
readonly props?: HiveJsonInputFormatProps;
constructor(props?: HiveJsonInputFormatProps);
private createHiveJsonSerde;
createInputFormatConfig(): CfnDeliveryStream.InputFormatConfigurationProperty;
}
/**
* Represents possible input formats when performing record data conversion.
*/
export declare class InputFormat {
/**
* Parse input JSON with OpenX JSON specification. This will typically suffice.
*/
static readonly OPENX_JSON: OpenXJsonInputFormat;
/**
* Parse input JSON with Hive JSON specification.
*/
static readonly HIVE_JSON: HiveJsonInputFormat;
private constructor();
}

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.InputFormat=exports.HiveJsonInputFormat=exports.TimestampParser=exports.OpenXJsonInputFormat=void 0;var jsiiDeprecationWarnings=()=>{var tmp=require("../../../.warnings.jsii.js");return jsiiDeprecationWarnings=()=>tmp,tmp};const JSII_RTTI_SYMBOL_1=Symbol.for("jsii.rtti");var cdk=()=>{var tmp=require("../../../core");return cdk=()=>tmp,tmp},literal_string_1=()=>{var tmp=require("../../../core/lib/private/literal-string");return literal_string_1=()=>tmp,tmp};class OpenXJsonInputFormat{static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.OpenXJsonInputFormat",version:"2.252.0"};props;constructor(props){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesisfirehose_OpenXJsonInputFormatProps(props)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,OpenXJsonInputFormat),error}this.props=props}createOpenXJsonSerde(){const props=this.props;return props?{caseInsensitive:props.lowercaseColumnNames,columnToJsonKeyMappings:props.columnToJsonKeyMappings,convertDotsInJsonKeysToUnderscores:props.convertDotsInJsonKeysToUnderscores}:{}}createInputFormatConfig(){return{deserializer:{openXJsonSerDe:this.createOpenXJsonSerde()}}}}exports.OpenXJsonInputFormat=OpenXJsonInputFormat;class TimestampParser{static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.TimestampParser",version:"2.252.0"};static EPOCH_MILLIS=new TimestampParser("millis");static fromFormatString(format){if(format===this.EPOCH_MILLIS.format)throw new(cdk()).UnscopedValidationError((0,literal_string_1().lit)`ReservedFormatStringNotAllowed`,`Cannot use reserved format string ${format} - Use 'TimestampParser.EPOCH_MILLIS' instead`);if(format.trim()==="")throw new(cdk()).UnscopedValidationError((0,literal_string_1().lit)`FormatStringCannotBeBlank`,"Format string cannot be blank or empty");return new TimestampParser(format)}format;constructor(format){this.format=format}}exports.TimestampParser=TimestampParser;class HiveJsonInputFormat{static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.HiveJsonInputFormat",version:"2.252.0"};props;constructor(props){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesisfirehose_HiveJsonInputFormatProps(props)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,HiveJsonInputFormat),error}this.props=props}createHiveJsonSerde(){const props=this.props;return props?{timestampFormats:props.timestampParsers?.map(parser=>parser.format)}:{}}createInputFormatConfig(){return{deserializer:{hiveJsonSerDe:this.createHiveJsonSerde()}}}}exports.HiveJsonInputFormat=HiveJsonInputFormat;class InputFormat{static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.InputFormat",version:"2.252.0"};static OPENX_JSON=new OpenXJsonInputFormat;static HIVE_JSON=new HiveJsonInputFormat;constructor(){}}exports.InputFormat=InputFormat;

View File

@@ -0,0 +1,304 @@
import * as cdk from '../../../core';
import type { CfnDeliveryStream } from '../kinesisfirehose.generated';
/**
* An output format to be used in Firehose record format conversion.
*/
export interface IOutputFormat {
/**
* Renders the cloudformation properties for the output format.
*/
createOutputFormatConfig(): CfnDeliveryStream.OutputFormatConfigurationProperty;
}
/**
* Possible compression options available for Parquet OutputFormat
*
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-parquetserde.html#cfn-kinesisfirehose-deliverystream-parquetserde-compression
*/
export declare class ParquetCompression {
readonly value: string;
/**
* Gzip
*/
static readonly GZIP: ParquetCompression;
/**
* Snappy
*/
static readonly SNAPPY: ParquetCompression;
/**
* Uncompressed
*/
static readonly UNCOMPRESSED: ParquetCompression;
/**
* Creates a new ParquetCompression instance with a custom value.
*/
static of(value: string): ParquetCompression;
/**
* @param value the string value of the Serde Compression.
*/
private constructor();
}
/**
* The available WriterVersions for Parquet output format
*/
export declare enum ParquetWriterVersion {
/**
* Use V1 Parquet writer version when writing the output
*/
V1 = "V1",
/**
* Use V2 Parquet writer version when writing the output
*/
V2 = "V2"
}
/**
* Props for Parquet output format for data record format conversion
*/
export interface ParquetOutputFormatProps {
/**
* The Hadoop Distributed File System (HDFS) block size.
* This is useful if you intend to copy the data from Amazon S3 to HDFS before querying.
* Firehose uses this value for padding calculations.
*
* @minimum `Size.mebibytes(64)`
* @default `Size.mebibytes(256)`
*/
readonly blockSize?: cdk.Size;
/**
* The compression code to use over data blocks.
*
* The possible values are `UNCOMPRESSED` , `SNAPPY` , and `GZIP`.
* Use `SNAPPY` for higher decompression speed.
* Use `GZIP` if the compression ratio is more important than speed.
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-parquetserde.html#cfn-kinesisfirehose-deliverystream-parquetserde-compression
* @default `SNAPPY`
*/
readonly compression?: ParquetCompression;
/**
* Indicates whether to enable dictionary compression.
*
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-parquetserde.html#cfn-kinesisfirehose-deliverystream-parquetserde-enabledictionarycompression
* @default `false`
*/
readonly enableDictionaryCompression?: boolean;
/**
* The maximum amount of padding to apply.
*
* This is useful if you intend to copy the data from Amazon S3 to HDFS before querying.
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-parquetserde.html#cfn-kinesisfirehose-deliverystream-parquetserde-maxpaddingbytes
* @default no padding is applied
*/
readonly maxPadding?: cdk.Size;
/**
* The Parquet page size.
*
* Column chunks are divided into pages. A page is conceptually an indivisible unit (in terms of compression and encoding). The minimum value is 64 KiB and the default is 1 MiB.
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-parquetserde.html#cfn-kinesisfirehose-deliverystream-parquetserde-pagesizebytes
*
* @minimum `Size.kibibytes(64)`
* @default `Size.mebibytes(1)`
*/
readonly pageSize?: cdk.Size;
/**
* Indicates the version of Parquet to output.
*
* The possible values are `V1` and `V2`
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-parquetserde.html#cfn-kinesisfirehose-deliverystream-parquetserde-writerversion
*
* @default `V1`
*/
readonly writerVersion?: ParquetWriterVersion;
}
/**
* This class specifies properties for Parquet output format for record format conversion.
*
* You should only need to specify an instance of this class if the default configuration does not suit your needs.
*/
export declare class ParquetOutputFormat implements IOutputFormat {
/**
* Properties for the Parquet output format
*/
readonly props?: ParquetOutputFormatProps;
constructor(props?: ParquetOutputFormatProps);
private validateProps;
private createParquetSerDeProps;
createOutputFormatConfig(): CfnDeliveryStream.OutputFormatConfigurationProperty;
}
/**
* Possible compression options available for ORC OutputFormat
*
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-kinesisfirehose-deliverystream-orcserde.html#cfn-kinesisfirehose-deliverystream-orcserde-compression
*/
export declare class OrcCompression {
readonly value: string;
/**
* Gzip
*/
static readonly ZLIB: OrcCompression;
/**
* Snappy
*/
static readonly SNAPPY: OrcCompression;
/**
* Uncompressed
*/
static readonly NONE: OrcCompression;
/**
* Creates a new OrcCompression instance with a custom value.
*/
static of(value: string): OrcCompression;
/**
* @param value the string value of the Serde Compression.
*/
private constructor();
}
/**
* The available WriterVersions for ORC output format
*/
export declare enum OrcFormatVersion {
/**
* Use V0_11 ORC writer version when writing the output of the record transformation
*/
V0_11 = "V0_11",
/**
* Use V0_12 ORC writer version when writing the output of the record transformation
*/
V0_12 = "V0_12"
}
/**
* Props for ORC output format for data record format conversion
*/
export interface OrcOutputFormatProps {
/**
* The Hadoop Distributed File System (HDFS) block size.
* This is useful if you intend to copy the data from Amazon S3 to HDFS before querying.
* Firehose uses this value for padding calculations.
*
* @minimum `Size.mebibytes(64)`
* @default `Size.mebibytes(256)`
*/
readonly blockSize?: cdk.Size;
/**
* The compression code to use over data blocks.
*
* The possible values are `NONE` , `SNAPPY` , and `ZLIB`.
* Use `SNAPPY` for higher decompression speed.
* Use `GZIP` if the compression ratio is more important than speed.
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/TemplateReference/aws-properties-kinesisfirehose-deliverystream-orcserde.html#cfn-kinesisfirehose-deliverystream-orcserde-compression
* @default `SNAPPY`
*/
readonly compression?: OrcCompression;
/**
* The column names for which you want Firehose to create bloom filters.
*
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-orcserde.html#cfn-kinesisfirehose-deliverystream-orcserde-bloomfiltercolumns
*
* @default no bloom filters are created
*/
readonly bloomFilterColumns?: string[];
/**
* The Bloom filter false positive probability (FPP).
*
* The lower the FPP, the bigger the bloom filter.
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-orcserde.html#cfn-kinesisfirehose-deliverystream-orcserde-bloomfilterfalsepositiveprobability
*
* @minimum `0`
* @maximum `1`
* @default `0.05`
*/
readonly bloomFilterFalsePositiveProbability?: number;
/**
* Determines whether dictionary encoding should be applied to a column.
*
* If the number of distinct keys (unique values) in a column exceeds this fraction of the total non-null rows in that column, dictionary encoding will be turned off for that specific column.
*
* To turn off dictionary encoding, set this threshold to 0. To always use dictionary encoding, set this threshold to 1.
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-orcserde.html#cfn-kinesisfirehose-deliverystream-orcserde-dictionarykeythreshold
*
* @minimum `0`
* @maximum `1`
* @default `0.8`
*/
readonly dictionaryKeyThreshold?: number;
/**
* Set this to `true` to indicate that you want stripes to be padded to the HDFS block boundaries.
*
* This is useful if you intend to copy the data from Amazon S3 to HDFS before querying.
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-orcserde.html#cfn-kinesisfirehose-deliverystream-orcserde-enablepadding
*
* @default `false`
*/
readonly enablePadding?: boolean;
/**
* The version of the ORC format to write.
*
* The possible values are `V0_11` and `V0_12`.
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-orcserde.html#cfn-kinesisfirehose-deliverystream-orcserde-formatversion
*
* @default `V0_12`
*/
readonly formatVersion?: OrcFormatVersion;
/**
* A number between 0 and 1 that defines the tolerance for block padding as a decimal fraction of stripe size.
*
* The default value is 0.05, which means 5 percent of stripe size.
*
* For the default values of 64 MiB ORC stripes and 256 MiB HDFS blocks, the default block padding tolerance of 5 percent reserves a maximum of 3.2 MiB for padding within the 256 MiB block.
* In such a case, if the available size within the block is more than 3.2 MiB, a new, smaller stripe is inserted to fit within that space.
* This ensures that no stripe crosses block boundaries and causes remote reads within a node-local task.
*
* Kinesis Data Firehose ignores this parameter when `EnablePadding` is `false` .
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-orcserde.html#cfn-kinesisfirehose-deliverystream-orcserde-paddingtolerance
*
* @default `0.05` if `enablePadding` is `true`
*/
readonly paddingTolerance?: number;
/**
* The number of rows between index entries.
*
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-orcserde.html#cfn-kinesisfirehose-deliverystream-orcserde-rowindexstride
*
* @minimum 1000
* @default 10000
*/
readonly rowIndexStride?: number;
/**
* The number of bytes in each stripe.
*
* The default is 64 MiB and the minimum is 8 MiB.
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-orcserde.html#cfn-kinesisfirehose-deliverystream-orcserde-stripesizebytes
*
* @minimum `Size.mebibytes(8)`
* @default `Size.mebibytes(64)`
*/
readonly stripeSize?: cdk.Size;
}
/**
* This class specifies properties for ORC output format for record format conversion.
*
* You should only need to specify an instance of this class if the default configuration does not suit your needs.
*/
export declare class OrcOutputFormat implements IOutputFormat {
/**
* Properties for the ORC output format
*/
readonly props?: OrcOutputFormatProps;
constructor(props?: OrcOutputFormatProps);
private betweenInclusive;
private validateProps;
private createOrcSerDeProps;
createOutputFormatConfig(): CfnDeliveryStream.OutputFormatConfigurationProperty;
}
/**
* Represents possible output formats when performing record data conversion.
*/
export declare class OutputFormat {
/**
* Write output files in Parquet
*/
static readonly PARQUET: ParquetOutputFormat;
/**
* Write output files in ORC
*/
static readonly ORC: OrcOutputFormat;
private constructor();
}

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,54 @@
import type { Construct } from 'constructs';
import type * as glue from '../../../aws-glue';
import * as iam from '../../../aws-iam';
import type { CfnDeliveryStream } from '../kinesisfirehose.generated';
/**
* Options when binding a SchemaConfig to a Destination
*/
export interface SchemaConfigurationBindOptions {
/**
* The IAM Role that will be used by the Delivery Stream for access to the Glue data catalog for record format conversion.
*/
readonly role: iam.IRole;
}
/**
* Options for creating a Schema for record format conversion from a `glue.CfnTable`
*/
export interface SchemaConfigurationFromCfnTableProps {
/**
* Specifies the table version for the output data schema.
*
* if set to `LATEST`, Firehose uses the most recent table version. This means that any updates to the table are automatically picked up.
*
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-schemaconfiguration.html#cfn-kinesisfirehose-deliverystream-schemaconfiguration-versionid
* @default `LATEST`
*/
readonly versionId?: string;
/**
* The region of the database the table is in.
*
* @default the region of the stack that contains the table reference is used
*/
readonly region?: string;
}
/**
* Represents a schema configuration for Firehose S3 data record format conversion.
*
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-dataformatconversionconfiguration.html#cfn-kinesisfirehose-deliverystream-dataformatconversionconfiguration-schemaconfiguration
*/
export declare class SchemaConfiguration {
private readonly databaseName;
private readonly tableName;
private readonly catalogId;
private readonly databaseRegion;
private readonly versionId;
/**
* Obtain schema configuration for data record format conversion from an `aws_glue.CfnTable`
*/
static fromCfnTable(table: glue.CfnTable, props?: SchemaConfigurationFromCfnTableProps): SchemaConfiguration;
private constructor();
/**
* Binds this Schema to the Destination, adding the necessary permissions to the Destination role.
*/
bind(scope: Construct, options: SchemaConfigurationBindOptions): CfnDeliveryStream.SchemaConfigurationProperty;
}

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.SchemaConfiguration=void 0;var jsiiDeprecationWarnings=()=>{var tmp=require("../../../.warnings.jsii.js");return jsiiDeprecationWarnings=()=>tmp,tmp};const JSII_RTTI_SYMBOL_1=Symbol.for("jsii.rtti");var iam=()=>{var tmp=require("../../../aws-iam");return iam=()=>tmp,tmp},cdk=()=>{var tmp=require("../../../core");return cdk=()=>tmp,tmp};class SchemaConfiguration{databaseName;tableName;catalogId;databaseRegion;versionId;static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.SchemaConfiguration",version:"2.252.0"};static fromCfnTable(table,props){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_glue_CfnTable(table),jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesisfirehose_SchemaConfigurationFromCfnTableProps(props)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,this.fromCfnTable),error}const stack=cdk().Stack.of(table);return new SchemaConfiguration(table.databaseName,table.ref,table.catalogId,props?.region??stack.region,props?.versionId??"LATEST")}constructor(databaseName,tableName,catalogId,databaseRegion,versionId){this.databaseName=databaseName,this.tableName=tableName,this.catalogId=catalogId,this.databaseRegion=databaseRegion,this.versionId=versionId}bind(scope,options){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesisfirehose_SchemaConfigurationBindOptions(options)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,this.bind),error}const stack=cdk().Stack.of(scope),catalogArn=stack.formatArn({service:"glue",resource:"catalog",region:this.databaseRegion,account:this.catalogId}),databaseArn=stack.formatArn({service:"glue",resource:"database",resourceName:this.databaseName,region:this.databaseRegion,account:this.catalogId}),tableArn=stack.formatArn({service:"glue",resource:"table",resourceName:`${this.databaseName}/${this.tableName}`,region:this.databaseRegion,account:this.catalogId});return iam().Grant.addToPrincipal({actions:["glue:GetTable","glue:GetTableVersion","glue:GetTableVersions"],grantee:options.role,resourceArns:[catalogArn,databaseArn,tableArn]}),iam().Grant.addToPrincipal({actions:["glue:GetSchemaVersion"],grantee:options.role,resourceArns:["*"]}),{roleArn:options.role.roleArn,region:this.databaseRegion,tableName:this.tableName,databaseName:this.databaseName,versionId:this.versionId,catalogId:this.catalogId}}}exports.SchemaConfiguration=SchemaConfiguration;

View File

@@ -0,0 +1,101 @@
import type { Construct } from 'constructs';
import type { CommonDestinationProps, CommonDestinationS3Props } from './common';
import type { DestinationBindOptions, DestinationConfig, IDestination } from './destination';
import type { IInputFormat, IOutputFormat, SchemaConfiguration } from './record-format';
import type * as s3 from '../../aws-s3';
import * as cdk from '../../core';
/**
* Props for defining an S3 destination of an Amazon Data Firehose delivery stream.
*/
export interface S3BucketProps extends CommonDestinationS3Props, CommonDestinationProps {
/**
* Specify a file extension.
* It will override the default file extension appended by Data Format Conversion or S3 compression features such as `.parquet` or `.gz`.
*
* File extension must start with a period (`.`) and can contain allowed characters: `0-9a-z!-_.*'()`.
*
* @see https://docs.aws.amazon.com/firehose/latest/dev/create-destination.html#create-destination-s3
* @default - The default file extension appended by Data Format Conversion or S3 compression features
*/
readonly fileExtension?: string;
/**
* The time zone you prefer.
*
* @see https://docs.aws.amazon.com/firehose/latest/dev/s3-prefixes.html#timestamp-namespace
*
* @default - UTC
*/
readonly timeZone?: cdk.TimeZone;
/**
* The input format, output format, and schema config for converting data from the JSON format to the Parquet or ORC format before writing to Amazon S3.
* @see https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-kinesisfirehose-deliverystream-extendeds3destinationconfiguration.html#cfn-kinesisfirehose-deliverystream-extendeds3destinationconfiguration-dataformatconversionconfiguration
*
* @default - no data format conversion is done
*/
readonly dataFormatConversion?: DataFormatConversionProps;
/**
* Specify dynamic partitioning.
* @see https://docs.aws.amazon.com/firehose/latest/dev/dynamic-partitioning.html
* @default - Dynamic partitioning is disabled.
*/
readonly dynamicPartitioning?: DynamicPartitioningProps;
}
/**
* Props for specifying data format conversion for Firehose
*
* @see https://docs.aws.amazon.com/firehose/latest/dev/record-format-conversion.html
*/
export interface DataFormatConversionProps {
/**
* Whether data format conversion is enabled or not.
*
* @default `true`
*/
readonly enabled?: boolean;
/**
* The schema configuration to use in converting the input format to output format
*/
readonly schemaConfiguration: SchemaConfiguration;
/**
* The input format to convert from for record format conversion
*/
readonly inputFormat: IInputFormat;
/**
* The output format to convert to for record format conversion
*/
readonly outputFormat: IOutputFormat;
}
/**
* Props for defining dynamic partitioning.
*
* @see https://docs.aws.amazon.com/firehose/latest/dev/dynamic-partitioning.html
*/
export interface DynamicPartitioningProps {
/**
* Whether to enable the dynamic partitioning.
*
* You cannot enable dynamic partitioning for an existing Firehose stream that does not have dynamic partitioning already enabled.
*
* @see https://docs.aws.amazon.com/firehose/latest/dev/dynamic-partitioning-enable.html
*/
readonly enabled: boolean;
/**
* The total amount of time that Data Firehose spends on retries.
*
* Minimum: Duration.seconds(0)
* Maximum: Duration.seconds(7200)
*
* @default Duration.seconds(300)
*/
readonly retryDuration?: cdk.Duration;
}
/**
* An S3 bucket destination for data from an Amazon Data Firehose delivery stream.
*/
export declare class S3Bucket implements IDestination {
private readonly bucket;
private readonly props;
constructor(bucket: s3.IBucket, props?: S3BucketProps);
bind(scope: Construct, _options: DestinationBindOptions): DestinationConfig;
private getS3BackupMode;
}

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,66 @@
import type { Construct } from 'constructs';
import type { CfnDeliveryStream } from './kinesisfirehose.generated';
import type * as iam from '../../aws-iam';
import type * as kinesis from '../../aws-kinesis';
/**
* An Amazon Data Firehose delivery stream source configuration.
*/
interface SourceConfig {
/**
* Configuration for using a Kinesis Data Stream as a source for the delivery stream.
*
* This will be returned by the _bind method depending on what type of Source class is specified.
*
* @default - Kinesis Data Stream Source configuration property is not provided.
*/
readonly kinesisStreamSourceConfiguration?: CfnDeliveryStream.KinesisStreamSourceConfigurationProperty;
/**
* Configuration for using an MSK (Managed Streaming for Kafka) cluster as a source for the delivery stream.
*
* This will be returned by the _bind method depending on what type of Source class is specified.
*
* @default - MSK Source configuration property is not provided.
*/
readonly mskSourceConfiguration?: CfnDeliveryStream.MSKSourceConfigurationProperty;
}
/**
* An interface for defining a source that can be used in an Amazon Data Firehose delivery stream.
*/
export interface ISource {
/**
* Binds this source to the Amazon Data Firehose delivery stream.
*
* @internal
*/
_bind(scope: Construct, roleArn?: string): SourceConfig;
/**
* Grant read permissions for this source resource and its contents to an IAM
* principal (the delivery stream).
*
* If an encryption key is used, permission to use the key to decrypt the
* contents of the stream will also be granted.
*/
grantRead(grantee: iam.IGrantable): iam.Grant;
}
/**
* An Amazon Data Firehose delivery stream source.
*/
export declare class KinesisStreamSource implements ISource {
private readonly stream;
/**
* Creates a new KinesisStreamSource.
*/
constructor(stream: kinesis.IStream);
/**
* [disable-awslint:no-grants]
*/
grantRead(grantee: iam.IGrantable): iam.Grant;
/**
* Binds the Kinesis stream as a source for the Amazon Data Firehose delivery stream.
*
* @returns The configuration needed to use this Kinesis stream as the delivery stream source.
* @internal
*/
_bind(_scope: Construct, roleArn: string): SourceConfig;
}
export {};

View File

@@ -0,0 +1 @@
"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.KinesisStreamSource=void 0;var jsiiDeprecationWarnings=()=>{var tmp=require("../../.warnings.jsii.js");return jsiiDeprecationWarnings=()=>tmp,tmp};const JSII_RTTI_SYMBOL_1=Symbol.for("jsii.rtti");class KinesisStreamSource{stream;static[JSII_RTTI_SYMBOL_1]={fqn:"aws-cdk-lib.aws_kinesisfirehose.KinesisStreamSource",version:"2.252.0"};constructor(stream){this.stream=stream;try{jsiiDeprecationWarnings().aws_cdk_lib_aws_kinesis_IStream(stream)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,KinesisStreamSource),error}}grantRead(grantee){try{jsiiDeprecationWarnings().aws_cdk_lib_aws_iam_IGrantable(grantee)}catch(error){throw process.env.JSII_DEBUG!=="1"&&error.name==="DeprecationError"&&Error.captureStackTrace(error,this.grantRead),error}return this.stream.grantRead(grantee)}_bind(_scope,roleArn){return{kinesisStreamSourceConfiguration:{kinesisStreamArn:this.stream.streamArn,roleArn}}}}exports.KinesisStreamSource=KinesisStreamSource;