diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/components/AggregationKeyDemo/index.js b/calico-cloud_versioned_docs/version-24-1/_includes/components/AggregationKeyDemo/index.js new file mode 100644 index 0000000000..d32cc4c54c --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/components/AggregationKeyDemo/index.js @@ -0,0 +1,172 @@ +import React, { useState, useMemo, useEffect, useRef } from 'react'; +import styles from './styles.module.css'; + +const cx = (...names) => names.filter(Boolean).join(' '); + +// HTTP-only sample so every aggregation toggle maps to a real FelixConfiguration +// field, keeping the rendered configuration block accurate. +const SAMPLE = [ + { src: 'frontend', dest: 'cart', method: 'GET', url: '/cart', code: '200' }, + { src: 'frontend', dest: 'cart', method: 'GET', url: '/cart', code: '404' }, + { src: 'frontend', dest: 'cart', method: 'GET', url: '/items', code: '200' }, + { src: 'frontend', dest: 'cart', method: 'POST', url: '/cart', code: '200' }, + { src: 'orders', dest: 'pay', method: 'POST', url: '/charge', code: '200' }, + { src: 'edge', dest: 'pay', method: 'POST', url: '/charge', code: '200' }, + { src: 'orders', dest: 'canary', method: 'POST', url: '/charge', code: '200' }, +]; + +// Each aggregator toggles one field in the aggregation key and maps to a real +// FelixConfiguration field with its include/exclude values. +const AGGREGATORS = [ + { id: 'source', label: 'Source info', field: 'src', yaml: 'l7LogsFileAggregationSourceInfo', inc: 'IncludeL7SourceInfo', exc: 'ExcludeL7SourceInfo' }, + { id: 'dest', label: 'Destination info', field: 'dest', yaml: 'l7LogsFileAggregationDestinationInfo', inc: 'IncludeL7DestinationInfo', exc: 'ExcludeL7DestinationInfo' }, + { id: 'method', label: 'HTTP method', field: 'method', yaml: 'l7LogsFileAggregationHTTPMethod', inc: 'IncludeL7HTTPMethod', exc: 'ExcludeL7HTTPMethod' }, + { id: 'url', label: 'URL', field: 'url', yaml: 'l7LogsFileAggregationTrimURL', inc: 'IncludeL7FullURL', exc: 'ExcludeL7URL' }, + { id: 'code', label: 'Response code', field: 'code', yaml: 'l7LogsFileAggregationResponseCode', inc: 'IncludeL7ResponseCode', exc: 'ExcludeL7ResponseCode' }, +]; + +// Maps each raw field to the log key it surfaces under, in log output order. +const LOG_KEYS = [ + { id: 'source', field: 'src', key: 'src_name_aggr' }, + { id: 'dest', field: 'dest', key: 'dest_name_aggr' }, + { id: 'method', field: 'method', key: 'method' }, + { id: 'url', field: 'url', key: 'url' }, + { id: 'code', field: 'code', key: 'response_code' }, +]; + +const RAW_COLUMNS = LOG_KEYS.map((k) => ({ field: k.field, label: k.key })); +const STATIC_CONFIG = [ + 'apiVersion: projectcalico.org/v3', + 'kind: FelixConfiguration', + 'metadata:', + ' name: default', + 'spec:', + ' l7LogsFileEnabled: true', +]; +const HIGHLIGHT_MS = 1500; + +export default function AggregationKeyDemo() { + const [checked, setChecked] = useState(() => + Object.fromEntries(AGGREGATORS.map((a) => [a.id, true])), + ); + + const toggle = (id) => setChecked((c) => ({ ...c, [id]: !c[id] })); + + // Aggregate the raw requests into buckets keyed by the included fields. + const logEntries = useMemo(() => { + const order = []; + const buckets = new Map(); + for (const row of SAMPLE) { + const keyParts = AGGREGATORS + .filter((a) => checked[a.id]) + .map((a) => `${a.field}:${row[a.field]}`); + const k = keyParts.join('|'); + if (!buckets.has(k)) { + const entry = { count: 0 }; + for (const lk of LOG_KEYS) { + entry[lk.key] = checked[lk.id] ? row[lk.field] : '-'; + } + buckets.set(k, entry); + order.push(k); + } + buckets.get(k).count += 1; + } + return order.map((k) => buckets.get(k)); + }, [checked]); + + const logText = useMemo( + () => logEntries.map((e) => JSON.stringify(e)).join('\n'), + [logEntries], + ); + + // Highlight the config lines whose include/exclude value just changed. + const prevChecked = useRef(checked); + const [highlighted, setHighlighted] = useState(() => new Set()); + useEffect(() => { + const changed = new Set( + AGGREGATORS.map((a) => a.id).filter((id) => checked[id] !== prevChecked.current[id]), + ); + prevChecked.current = checked; + if (changed.size === 0) return; + setHighlighted(changed); + const t = setTimeout(() => setHighlighted(new Set()), HIGHLIGHT_MS); + return () => clearTimeout(t); + }, [checked]); + + return ( +
+

1. Observed traffic ({SAMPLE.length} requests)

+

+ Consider these {SAMPLE.length} requests as the raw L7 events captured + between workloads in your cluster — one row per request, exactly as seen + on the wire before any aggregation is applied. This is the input that the + aggregation key collapses into log entries. +

+
+ + + + {RAW_COLUMNS.map((c) => ( + + ))} + + + + {SAMPLE.map((row, i) => ( + + {RAW_COLUMNS.map((c) => ( + + ))} + + ))} + +
{c.label}
{row[c.field]}
+
+ +

2. Aggregation key fields

+

+ Include or exclude each field from the aggregation key. +

+
+ {AGGREGATORS.map((a) => ( + + ))} +
+ +

3. Configuration

+

+ The FelixConfiguration produced by the selected fields. +

+ 
+        {STATIC_CONFIG.map((line) => (
+          
{line}

+        ))}
+        {AGGREGATORS.map((a) => (
+          

+            {`  ${a.yaml}: ${checked[a.id] ? a.inc : a.exc}`}
+          

+        ))}
+
+ +

+ 4. Resulting log ({logEntries.length} {logEntries.length === 1 ? 'entry' : 'entries'}) +

+

+ What is written to the log file: one JSON entry per unique combination of + the included fields, with count requests merged into it. + Excluded fields collapse to "-". +

+ 
{logText}
+
+ ); +} diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/components/AggregationKeyDemo/styles.module.css b/calico-cloud_versioned_docs/version-24-1/_includes/components/AggregationKeyDemo/styles.module.css new file mode 100644 index 0000000000..a5d853174e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/components/AggregationKeyDemo/styles.module.css @@ -0,0 +1,73 @@ +.controls { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px 16px; + margin: 12px 0; + padding: 12px; + border: 1px solid var(--ifm-color-emphasis-300); + border-radius: 4px; +} + +.controlsTitle { + font-weight: 600; +} + +.controlLabel { + display: inline-flex; + align-items: center; + gap: 4px; + cursor: pointer; +} + +.sectionTitle { + margin: 20px 0 2px; + font-weight: 600; +} + +.sectionHint { + margin: 0 0 8px; + font-size: 0.85em; + opacity: 0.8; +} + +.tableWrap { + overflow-x: auto; +} + +.table { + width: 100%; +} + +.cell { + padding: 4px 8px; + font-family: var(--ifm-font-family-monospace); + font-size: 0.85em; + overflow-wrap: anywhere; + vertical-align: top; +} + +.header { + padding: 4px 8px; + font-family: var(--ifm-font-family-monospace); + font-size: 0.85em; + white-space: normal; + word-break: break-word; +} + +.code { + overflow-x: auto; + white-space: pre; + font-size: 0.85em; +} + +.codeLine { + display: block; + border-radius: 2px; + transition: background-color 1s ease; +} + +.codeLineHi { + background-color: var(--ifm-color-success-contrast-background); + transition: background-color 0s; +} diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/components/EnvironmentFile.js b/calico-cloud_versioned_docs/version-24-1/_includes/components/EnvironmentFile.js new file mode 100644 index 0000000000..a706a0c74e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/components/EnvironmentFile.js @@ -0,0 +1,59 @@ +import React from 'react'; + +import Admonition from '@theme/Admonition'; +import CodeBlock from '@theme/CodeBlock'; +import Link from '@docusaurus/Link'; + +import { baseUrl } from '../../variables'; + +export default function EnvironmentFile(props) { + return ( + <> +

+ {props.install === 'container' ? ( + + Use the following guidelines and sample file to define the environment variables for starting Calico on the host. + For more help, see the {props.nodecontainer} configuration reference + + ) : ( + + Use the following guidelines and sample file to define the environment variables for starting Calico on the host. + + )} +

+

For the Kubernetes datastore set the following:

+ + + + + + + + + + + + + +
VariableConfiguration guidance
KUBECONFIGPath to kubeconfig file to access the Kubernetes API Server
+ {props.install === 'container' && ( + + If using certificates and keys, you will need to volume mount them into the container at the location + specified by the paths mentioned above. + + )} +

+ Sample EnvironmentFile - save to /etc/calico/calico.env +

+ + {`DATASTORE_TYPE=kubernetes +CALICO_NODENAME="" +NO_DEFAULT_POOLS="true" +CALICO_IP="" +CALICO_IP6="" +CALICO_AS="" +CALICO_NETWORKING_BACKEND=bird`} + + + ); +} diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/TableConfig.js b/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/TableConfig.js new file mode 100644 index 0000000000..b0d6ba7edf --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/TableConfig.js @@ -0,0 +1,55 @@ +import React from 'react'; +import sanitizeHtml from 'sanitize-html'; +import styles from './styles.module.css'; + +const getSanitizedData = ({ fieldData }) => ({ + sanitizedDescription: { __html: sanitizeHtml(fieldData.DescriptionHTML) }, + sanitizedSchema: { __html: sanitizeHtml(fieldData.StringSchemaHTML) }, +}); + +const TableConfig = ({ fieldData }) => { + const { sanitizedDescription, sanitizedSchema } = getSanitizedData({ fieldData }); + + return ( + + + + + + + + + + + + + + + + + + + + + + + +
AttributeValue
Key + {fieldData.NameConfigFile || 'No Default Value'} +
Description +
Schema +
Default + {fieldData.StringDefault === '' ? ( + 'none' + ) : fieldData.GoType === '*v1.Duration' ? ( + <> + {fieldData.StringDefault} ({fieldData.ParsedDefault}) + + ) : ( + {fieldData.StringDefault} + )} +
+ ); +}; + +export default TableConfig; diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/TableEnv.js b/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/TableEnv.js new file mode 100644 index 0000000000..e71bbe32f8 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/TableEnv.js @@ -0,0 +1,55 @@ +import React from 'react'; +import sanitizeHtml from 'sanitize-html'; +import styles from './styles.module.css'; + +const getSanitizedData = ({ fieldData }) => ({ + sanitizedDescription: { __html: sanitizeHtml(fieldData.DescriptionHTML) }, + sanitizedSchema: { __html: sanitizeHtml(fieldData.StringSchemaHTML) }, +}); + +const TableEnv = ({ fieldData }) => { + const { sanitizedDescription, sanitizedSchema } = getSanitizedData({ fieldData }); + + return ( + + + + + + + + + + + + + + + + + + + + + + + +
AttributeValue
Key + {fieldData.NameEnvVar.toUpperCase() || 'No Default Value'} +
Description +
Schema +
Default + {fieldData.StringDefault === '' ? ( + 'none' + ) : fieldData.GoType === '*v1.Duration' ? ( + <> + {fieldData.StringDefault} ({fieldData.ParsedDefault}) + + ) : ( + {fieldData.StringDefault} + )} +
+ ); +}; + +export default TableEnv; diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/TableResource.js b/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/TableResource.js new file mode 100644 index 0000000000..147333df4e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/TableResource.js @@ -0,0 +1,46 @@ +import React from 'react'; +import sanitizeHtml from 'sanitize-html'; +import styles from './styles.module.css'; + +const getSanitizedData = ({ fieldData }) => ({ + sanitizedNAMEYAML: { __html: sanitizeHtml(fieldData.NameYAML) }, + sanitizedDescription: { __html: sanitizeHtml(fieldData.DescriptionHTML) }, + sanitizedSchema: { __html: sanitizeHtml(fieldData.YAMLSchemaHTML) }, +}); + +const TableResource = ({ fieldData }) => { + const { sanitizedNAMEYAML, sanitizedDescription, sanitizedSchema } = getSanitizedData({ fieldData }); + + return ( + + + + + + + + + + + + + + + + + + + + + + + +
AttributeValue
Key + +
Description +
Schema +
Default{fieldData.YAMLDefault === '' ? 'none' : {fieldData.YAMLDefault}}
+ ); +}; + +export default TableResource; diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/config-params.json b/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/config-params.json new file mode 100644 index 0000000000..5024a1c18e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/config-params.json @@ -0,0 +1,9377 @@ +{ + "Comment": "This file generated by calico-felix-docgen, DO NOT EDIT.", + "Groups": [ + { + "Name": "Datastore connection", + "Fields": [ + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "DatastoreType", + "NameEnvVar": "FELIX_DatastoreType", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "One of: `etcdv3`, `kubernetes`", + "StringSchemaHTML": "One of: etcdv3, kubernetes", + "StringDefault": "etcdv3", + "ParsedDefault": "etcdv3", + "ParsedDefaultJSON": "\"etcdv3\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "LocalOnly", + "Description": "Controls which datastore driver Felix will use. Typically, this is detected from the environment\nand it does not need to be set manually. (For example, if `KUBECONFIG` is set, the kubernetes datastore driver\nwill be used by default).", + "DescriptionHTML": "

Controls which datastore driver Felix will use. Typically, this is detected from the environment\nand it does not need to be set manually. (For example, if KUBECONFIG is set, the kubernetes datastore driver\nwill be used by default).

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "EtcdAddr", + "NameEnvVar": "FELIX_EtcdAddr", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String matching regex `^[^:/]+:\\d+$`", + "StringSchemaHTML": "String matching regex ^[^:/]+:\\d+$", + "StringDefault": "127.0.0.1:2379", + "ParsedDefault": "127.0.0.1:2379", + "ParsedDefaultJSON": "\"127.0.0.1:2379\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "**Open source-only parameter**; `etcdv3` datastore driver is not supported in Calico Enterprise/Cloud.\n\nWhen using the `etcdv3` datastore driver, the etcd server and port to connect to. If EtcdEndpoints\nis also specified, it takes precedence.", + "DescriptionHTML": "

Open source-only parameter; etcdv3 datastore driver is not supported in Calico Enterprise/Cloud.

\n

When using the etcdv3 datastore driver, the etcd server and port to connect to. If EtcdEndpoints\nis also specified, it takes precedence.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": true + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "EtcdCaFile", + "NameEnvVar": "FELIX_EtcdCaFile", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "**Open source-only parameter**; `etcdv3` datastore driver is not supported in Calico Enterprise/Cloud.\n\nWhen using the `etcdv3` datastore driver, path to TLS CA file to use when connecting to\netcd. If the CA file is specified, the other TLS parameters are mandatory.", + "DescriptionHTML": "

Open source-only parameter; etcdv3 datastore driver is not supported in Calico Enterprise/Cloud.

\n

When using the etcdv3 datastore driver, path to TLS CA file to use when connecting to\netcd. If the CA file is specified, the other TLS parameters are mandatory.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": true + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "EtcdCertFile", + "NameEnvVar": "FELIX_EtcdCertFile", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "**Open source-only parameter**; `etcdv3` datastore driver is not supported in Calico Enterprise/Cloud.\n\nWhen using the `etcdv3` datastore driver, path to TLS certificate file to use when connecting to\netcd. If the certificate file is specified, the other TLS parameters are mandatory.", + "DescriptionHTML": "

Open source-only parameter; etcdv3 datastore driver is not supported in Calico Enterprise/Cloud.

\n

When using the etcdv3 datastore driver, path to TLS certificate file to use when connecting to\netcd. If the certificate file is specified, the other TLS parameters are mandatory.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": true + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "EtcdEndpoints", + "NameEnvVar": "FELIX_EtcdEndpoints", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "List of HTTP endpoints: comma-delimited list of `http(s)://hostname:port`", + "StringSchemaHTML": "List of HTTP endpoints: comma-delimited list of http(s)://hostname:port", + "StringDefault": "", + "ParsedDefault": "[]", + "ParsedDefaultJSON": "null", + "ParsedType": "[]string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "**Open source-only parameter**; `etcdv3` datastore driver is not supported in Calico Enterprise/Cloud.\n\nWhen using the `etcdv3` datastore driver, comma-delimited list of etcd endpoints to connect to,\nreplaces EtcdAddr and EtcdScheme.", + "DescriptionHTML": "

Open source-only parameter; etcdv3 datastore driver is not supported in Calico Enterprise/Cloud.

\n

When using the etcdv3 datastore driver, comma-delimited list of etcd endpoints to connect to,\nreplaces EtcdAddr and EtcdScheme.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": true + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "EtcdKeyFile", + "NameEnvVar": "FELIX_EtcdKeyFile", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "**Open source-only parameter**; `etcdv3` datastore driver is not supported in Calico Enterprise/Cloud.\n\nWhen using the `etcdv3` datastore driver, path to TLS private key file to use when connecting to\netcd. If the key file is specified, the other TLS parameters are mandatory.", + "DescriptionHTML": "

Open source-only parameter; etcdv3 datastore driver is not supported in Calico Enterprise/Cloud.

\n

When using the etcdv3 datastore driver, path to TLS private key file to use when connecting to\netcd. If the key file is specified, the other TLS parameters are mandatory.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": true + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "EtcdScheme", + "NameEnvVar": "FELIX_EtcdScheme", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "One of: `http`, `https`", + "StringSchemaHTML": "One of: http, https", + "StringDefault": "http", + "ParsedDefault": "http", + "ParsedDefaultJSON": "\"http\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "**Open source-only parameter**; `etcdv3` datastore driver is not supported in Calico Enterprise/Cloud.\n\nEtcdAddr: when using the `etcdv3` datastore driver, the URL scheme to use. If EtcdEndpoints\nis also specified, it takes precedence.", + "DescriptionHTML": "

Open source-only parameter; etcdv3 datastore driver is not supported in Calico Enterprise/Cloud.

\n

EtcdAddr: when using the etcdv3 datastore driver, the URL scheme to use. If EtcdEndpoints\nis also specified, it takes precedence.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": true + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "FelixHostname", + "NameEnvVar": "FELIX_FelixHostname", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String matching regex `^[a-zA-Z0-9_.-]+$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9_.-]+$", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "The name of this node, used to identify resources in the datastore that belong to this node.\nAuto-detected from the node's hostname if not provided.", + "DescriptionHTML": "

The name of this node, used to identify resources in the datastore that belong to this node.\nAuto-detected from the node's hostname if not provided.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaAddr", + "NameEnvVar": "FELIX_TyphaAddr", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String matching regex `^[^:/]+:\\d+$`", + "StringSchemaHTML": "String matching regex ^[^:/]+:\\d+$", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "If set, tells Felix to connect to Typha at the given address and port. Overrides TyphaK8sServiceName.", + "DescriptionHTML": "

If set, tells Felix to connect to Typha at the given address and port. Overrides TyphaK8sServiceName.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaCAFile", + "NameEnvVar": "FELIX_TyphaCAFile", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Path to the TLS CA file to use when communicating with Typha. If this parameter is specified,\nthe other TLS parameters must also be specified. For non-cluster hosts, the CA file is extracted from the\ntigera-ca-bundle ConfigMap under the TyphaK8sNamespace namespace.", + "DescriptionHTML": "

Path to the TLS CA file to use when communicating with Typha. If this parameter is specified,\nthe other TLS parameters must also be specified. For non-cluster hosts, the CA file is extracted from the\ntigera-ca-bundle ConfigMap under the TyphaK8sNamespace namespace.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaCN", + "NameEnvVar": "FELIX_TyphaCN", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Common name to use when authenticating to Typha over TLS. If any TLS parameters are specified then one of\nTyphaCN and TyphaURISAN must be set.", + "DescriptionHTML": "

Common name to use when authenticating to Typha over TLS. If any TLS parameters are specified then one of\nTyphaCN and TyphaURISAN must be set.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaCertFile", + "NameEnvVar": "FELIX_TyphaCertFile", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Path to the TLS certificate to use when communicating with Typha. If this parameter is specified,\nthe other TLS parameters must also be specified. For non-cluster hosts, the certificate will be signed by the\nin-cluster Tigera operator signer.", + "DescriptionHTML": "

Path to the TLS certificate to use when communicating with Typha. If this parameter is specified,\nthe other TLS parameters must also be specified. For non-cluster hosts, the certificate will be signed by the\nin-cluster Tigera operator signer.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaK8sNamespace", + "NameEnvVar": "FELIX_TyphaK8sNamespace", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "kube-system", + "ParsedDefault": "kube-system", + "ParsedDefaultJSON": "\"kube-system\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Namespace to look in when looking for Typha's service (see TyphaK8sServiceName).", + "DescriptionHTML": "

Namespace to look in when looking for Typha's service (see TyphaK8sServiceName).

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaK8sServiceName", + "NameEnvVar": "FELIX_TyphaK8sServiceName", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "If set, tells Felix to connect to Typha by looking up the Endpoints of the given Kubernetes\nService in namespace specified by TyphaK8sNamespace.", + "DescriptionHTML": "

If set, tells Felix to connect to Typha by looking up the Endpoints of the given Kubernetes\nService in namespace specified by TyphaK8sNamespace.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaKeyFile", + "NameEnvVar": "FELIX_TyphaKeyFile", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Path to the TLS private key to use when communicating with Typha. If this parameter is specified,\nthe other TLS parameters must also be specified. For non-cluster hosts, the private key is generated locally\nand rotated when the certificate expires.", + "DescriptionHTML": "

Path to the TLS private key to use when communicating with Typha. If this parameter is specified,\nthe other TLS parameters must also be specified. For non-cluster hosts, the private key is generated locally\nand rotated when the certificate expires.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaReadTimeout", + "NameEnvVar": "FELIX_TyphaReadTimeout", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "30", + "ParsedDefault": "30s", + "ParsedDefaultJSON": "30000000000", + "ParsedType": "time.Duration", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Read timeout when reading from the Typha connection. If typha sends no data for this long,\nFelix will exit and restart. (Note that Typha sends regular pings so traffic is always expected.)", + "DescriptionHTML": "

Read timeout when reading from the Typha connection. If typha sends no data for this long,\nFelix will exit and restart. (Note that Typha sends regular pings so traffic is always expected.)

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaURISAN", + "NameEnvVar": "FELIX_TyphaURISAN", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "URI SAN to use when authenticating to Typha over TLS. If any TLS parameters are specified then one of\nTyphaCN and TyphaURISAN must be set.", + "DescriptionHTML": "

URI SAN to use when authenticating to Typha over TLS. If any TLS parameters are specified then one of\nTyphaCN and TyphaURISAN must be set.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Datastore connection", + "GroupWithSortPrefix": "00 Datastore connection", + "NameConfigFile": "TyphaWriteTimeout", + "NameEnvVar": "FELIX_TyphaWriteTimeout", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "10", + "ParsedDefault": "10s", + "ParsedDefaultJSON": "10000000000", + "ParsedType": "time.Duration", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Write timeout when writing data to Typha.", + "DescriptionHTML": "

Write timeout when writing data to Typha.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Process: Feature detection/overrides", + "Fields": [ + { + "Group": "Process: Feature detection/overrides", + "GroupWithSortPrefix": "00 Process: Feature detection/overrides", + "NameConfigFile": "FeatureDetectOverride", + "NameEnvVar": "FELIX_FeatureDetectOverride", + "NameYAML": "featureDetectOverride", + "NameGoAPI": "FeatureDetectOverride", + "StringSchema": "Comma-delimited list of key=value pairs", + "StringSchemaHTML": "Comma-delimited list of key=value pairs", + "StringDefault": "", + "ParsedDefault": "map[]", + "ParsedDefaultJSON": "null", + "ParsedType": "map[string]string", + "YAMLType": "string", + "YAMLSchema": "String matching the regular expression `^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String matching the regular expression ^([a-zA-Z0-9-_]+=(true|false|),)*([a-zA-Z0-9-_]+=(true|false|))?$.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to override feature detection based on auto-detected platform\ncapabilities. Values are specified in a comma separated list with no spaces, example;\n\"SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=\". A value of \"true\" or \"false\" will\nforce enable/disable feature, empty or omitted values fall back to auto-detection.", + "DescriptionHTML": "

Used to override feature detection based on auto-detected platform\ncapabilities. Values are specified in a comma separated list with no spaces, example;\n\"SNATFullyRandom=true,MASQFullyRandom=false,RestoreSupportsLock=\". A value of \"true\" or \"false\" will\nforce enable/disable feature, empty or omitted values fall back to auto-detection.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Process: Feature detection/overrides", + "GroupWithSortPrefix": "00 Process: Feature detection/overrides", + "NameConfigFile": "FeatureGates", + "NameEnvVar": "FELIX_FeatureGates", + "NameYAML": "featureGates", + "NameGoAPI": "FeatureGates", + "StringSchema": "Comma-delimited list of key=value pairs", + "StringSchemaHTML": "Comma-delimited list of key=value pairs", + "StringDefault": "", + "ParsedDefault": "map[]", + "ParsedDefaultJSON": "null", + "ParsedType": "map[string]string", + "YAMLType": "string", + "YAMLSchema": "String matching the regular expression `^([a-zA-Z0-9-_]+=([^=]+),)*([a-zA-Z0-9-_]+=([^=]+))?$`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String matching the regular expression ^([a-zA-Z0-9-_]+=([^=]+),)*([a-zA-Z0-9-_]+=([^=]+))?$.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to enable or disable tech-preview Calico features.\nValues are specified in a comma separated list with no spaces, example;\n\"BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false\". This is\nused to enable features that are not fully production ready.", + "DescriptionHTML": "

Used to enable or disable tech-preview Calico features.\nValues are specified in a comma separated list with no spaces, example;\n\"BPFConnectTimeLoadBalancingWorkaround=enabled,XyZ=false\". This is\nused to enable features that are not fully production ready.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Process: Go runtime", + "Fields": [ + { + "Group": "Process: Go runtime", + "GroupWithSortPrefix": "00 Process: Go runtime", + "NameConfigFile": "GoGCThreshold", + "NameEnvVar": "FELIX_GoGCThreshold", + "NameYAML": "goGCThreshold", + "NameGoAPI": "GoGCThreshold", + "StringSchema": "Integer: [-1,2^63-1]", + "StringSchemaHTML": "Integer: [-1,263-1]", + "StringDefault": "40", + "ParsedDefault": "40", + "ParsedDefaultJSON": "40", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [-1,2^63-1]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [-1,263-1]", + "YAMLDefault": "40", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the Go runtime's garbage collection threshold. I.e. the percentage that the heap is\nallowed to grow before garbage collection is triggered. In general, doubling the value halves the CPU time\nspent doing GC, but it also doubles peak GC memory overhead. A special value of -1 can be used\nto disable GC entirely; this should only be used in conjunction with the GoMemoryLimitMB setting.\n\nThis setting is overridden by the GOGC environment variable.", + "DescriptionHTML": "

Sets the Go runtime's garbage collection threshold. I.e. the percentage that the heap is\nallowed to grow before garbage collection is triggered. In general, doubling the value halves the CPU time\nspent doing GC, but it also doubles peak GC memory overhead. A special value of -1 can be used\nto disable GC entirely; this should only be used in conjunction with the GoMemoryLimitMB setting.

\n

This setting is overridden by the GOGC environment variable.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Process: Go runtime", + "GroupWithSortPrefix": "00 Process: Go runtime", + "NameConfigFile": "GoMaxProcs", + "NameEnvVar": "FELIX_GoMaxProcs", + "NameYAML": "goMaxProcs", + "NameGoAPI": "GoMaxProcs", + "StringSchema": "Integer: [-1,2^63-1]", + "StringSchemaHTML": "Integer: [-1,263-1]", + "StringDefault": "-1", + "ParsedDefault": "-1", + "ParsedDefaultJSON": "-1", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [-1,2^63-1]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [-1,263-1]", + "YAMLDefault": "-1", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the maximum number of CPUs that the Go runtime will use concurrently. A value of -1 means\n\"use the system default\"; typically the number of real CPUs on the system.\n\nthis setting is overridden by the GOMAXPROCS environment variable.", + "DescriptionHTML": "

Sets the maximum number of CPUs that the Go runtime will use concurrently. A value of -1 means\n\"use the system default\"; typically the number of real CPUs on the system.

\n

this setting is overridden by the GOMAXPROCS environment variable.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Process: Go runtime", + "GroupWithSortPrefix": "00 Process: Go runtime", + "NameConfigFile": "GoMemoryLimitMB", + "NameEnvVar": "FELIX_GoMemoryLimitMB", + "NameYAML": "goMemoryLimitMB", + "NameGoAPI": "GoMemoryLimitMB", + "StringSchema": "Integer: [-1,2^63-1]", + "StringSchemaHTML": "Integer: [-1,263-1]", + "StringDefault": "-1", + "ParsedDefault": "-1", + "ParsedDefaultJSON": "-1", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [-1,2^63-1]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [-1,263-1]", + "YAMLDefault": "-1", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets a (soft) memory limit for the Go runtime in MB. The Go runtime will try to keep its memory\nusage under the limit by triggering GC as needed. To avoid thrashing, it will exceed the limit if GC starts to\ntake more than 50% of the process's CPU time. A value of -1 disables the memory limit.\n\nNote that the memory limit, if used, must be considerably less than any hard resource limit set at the container\nor pod level. This is because felix is not the only process that must run in the container or pod.\n\nThis setting is overridden by the GOMEMLIMIT environment variable.", + "DescriptionHTML": "

Sets a (soft) memory limit for the Go runtime in MB. The Go runtime will try to keep its memory\nusage under the limit by triggering GC as needed. To avoid thrashing, it will exceed the limit if GC starts to\ntake more than 50% of the process's CPU time. A value of -1 disables the memory limit.

\n

Note that the memory limit, if used, must be considerably less than any hard resource limit set at the container\nor pod level. This is because felix is not the only process that must run in the container or pod.

\n

This setting is overridden by the GOMEMLIMIT environment variable.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Process: Health port and timeouts", + "Fields": [ + { + "Group": "Process: Health port and timeouts", + "GroupWithSortPrefix": "00 Process: Health port and timeouts", + "NameConfigFile": "HealthEnabled", + "NameEnvVar": "FELIX_HealthEnabled", + "NameYAML": "healthEnabled", + "NameGoAPI": "HealthEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "If set to true, enables Felix's health port, which provides readiness and liveness endpoints.", + "DescriptionHTML": "

If set to true, enables Felix's health port, which provides readiness and liveness endpoints.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Process: Health port and timeouts", + "GroupWithSortPrefix": "00 Process: Health port and timeouts", + "NameConfigFile": "HealthHost", + "NameEnvVar": "FELIX_HealthHost", + "NameYAML": "healthHost", + "NameGoAPI": "HealthHost", + "StringSchema": "String matching regex `^[a-zA-Z0-9:._+-]{1,64}$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9:._+-]{1,64}$", + "StringDefault": "localhost", + "ParsedDefault": "localhost", + "ParsedDefaultJSON": "\"localhost\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "localhost", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The host that the health server should bind to.", + "DescriptionHTML": "

The host that the health server should bind to.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Process: Health port and timeouts", + "GroupWithSortPrefix": "00 Process: Health port and timeouts", + "NameConfigFile": "HealthPort", + "NameEnvVar": "FELIX_HealthPort", + "NameYAML": "healthPort", + "NameGoAPI": "HealthPort", + "StringSchema": "Integer: [0,65535]", + "StringSchemaHTML": "Integer: [0,65535]", + "StringDefault": "9099", + "ParsedDefault": "9099", + "ParsedDefaultJSON": "9099", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [0,65535]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [0,65535]", + "YAMLDefault": "9099", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The TCP port that the health server should bind to.", + "DescriptionHTML": "

The TCP port that the health server should bind to.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Process: Health port and timeouts", + "GroupWithSortPrefix": "00 Process: Health port and timeouts", + "NameConfigFile": "HealthTimeoutOverrides", + "NameEnvVar": "FELIX_HealthTimeoutOverrides", + "NameYAML": "healthTimeoutOverrides", + "NameGoAPI": "HealthTimeoutOverrides", + "StringSchema": "Comma-delimited list of `=` pairs, where durations use Go's standard format (e.g. 1s, 1m, 1h3m2s)", + "StringSchemaHTML": "Comma-delimited list of <key>=<duration> pairs, where durations use Go's standard format (e.g. 1s, 1m, 1h3m2s)", + "StringDefault": "", + "ParsedDefault": "map[]", + "ParsedDefaultJSON": "null", + "ParsedType": "map[string]time.Duration", + "YAMLType": "array", + "YAMLSchema": "List of health timeout overrides: `[{name: \"\", timeout: \"\"}, ...]` where `` is in the Go duration format, for example `1m30s`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of health timeout overrides: [{name: \"<name>\", timeout: \"<duration>\"}, ...] where <duration> is in the Go duration format, for example 1m30s.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Allows the internal watchdog timeouts of individual subcomponents to be\noverridden. This is useful for working around \"false positive\" liveness timeouts that can occur\nin particularly stressful workloads or if CPU is constrained. For a list of active\nsubcomponents, see Felix's logs.", + "DescriptionHTML": "

Allows the internal watchdog timeouts of individual subcomponents to be\noverridden. This is useful for working around \"false positive\" liveness timeouts that can occur\nin particularly stressful workloads or if CPU is constrained. For a list of active\nsubcomponents, see Felix's logs.

", + "UserEditable": true, + "GoType": "[]v3.HealthTimeoutOverride", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Process: Logging", + "Fields": [ + { + "Group": "Process: Logging", + "GroupWithSortPrefix": "00 Process: Logging", + "NameConfigFile": "LogActionRateLimit", + "NameEnvVar": "FELIX_LogActionRateLimit", + "NameYAML": "logActionRateLimit", + "NameGoAPI": "LogActionRateLimit", + "StringSchema": "String matching regex `^([1-9]\\d{0,3}/(?:second|minute|hour|day))?$`", + "StringSchemaHTML": "String matching regex ^([1-9]\\d{0,3}/(?:second|minute|hour|day))?$", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String matching the regular expression `^[1-9]\\d{0,3}/(?:second|minute|hour|day)$`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String matching the regular expression ^[1-9]\\d{0,3}/(?:second|minute|hour|day)$.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the rate of hitting a Log action. The value must be in the format \"N/unit\",\nwhere N is a number and unit is one of: second, minute, hour, or day. For example: \"10/second\" or \"100/hour\".", + "DescriptionHTML": "

Sets the rate of hitting a Log action. The value must be in the format \"N/unit\",\nwhere N is a number and unit is one of: second, minute, hour, or day. For example: \"10/second\" or \"100/hour\".

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Process: Logging", + "GroupWithSortPrefix": "00 Process: Logging", + "NameConfigFile": "LogActionRateLimitBurst", + "NameEnvVar": "FELIX_LogActionRateLimitBurst", + "NameYAML": "logActionRateLimitBurst", + "NameGoAPI": "LogActionRateLimitBurst", + "StringSchema": "Integer: [0,2^63-1], [9999,2^63-1]", + "StringSchemaHTML": "Integer: [0,263-1], [9999,263-1]", + "StringDefault": "5", + "ParsedDefault": "5", + "ParsedDefaultJSON": "5", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [0,2^63-1], [9999,2^63-1]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [0,263-1], [9999,263-1]", + "YAMLDefault": "5", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the rate limit burst of hitting a Log action when LogActionRateLimit is enabled.", + "DescriptionHTML": "

Sets the rate limit burst of hitting a Log action when LogActionRateLimit is enabled.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Process: Logging", + "GroupWithSortPrefix": "00 Process: Logging", + "NameConfigFile": "LogDebugFilenameRegex", + "NameEnvVar": "FELIX_LogDebugFilenameRegex", + "NameYAML": "logDebugFilenameRegex", + "NameGoAPI": "LogDebugFilenameRegex", + "StringSchema": "Regular expression", + "StringSchemaHTML": "Regular expression", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "null", + "ParsedType": "*regexp.Regexp", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls which source code files have their Debug log output included in the logs.\nOnly logs from files with names that match the given regular expression are included. The filter only applies\nto Debug level logs.", + "DescriptionHTML": "

Controls which source code files have their Debug log output included in the logs.\nOnly logs from files with names that match the given regular expression are included. The filter only applies\nto Debug level logs.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Process: Logging", + "GroupWithSortPrefix": "00 Process: Logging", + "NameConfigFile": "LogDropActionOverride", + "NameEnvVar": "FELIX_LogDropActionOverride", + "NameYAML": "logDropActionOverride", + "NameGoAPI": "LogDropActionOverride", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies whether or not to include the DropActionOverride in the logs when it is triggered.", + "DescriptionHTML": "

Specifies whether or not to include the DropActionOverride in the logs when it is triggered.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Process: Logging", + "GroupWithSortPrefix": "00 Process: Logging", + "NameConfigFile": "LogFilePath", + "NameEnvVar": "FELIX_LogFilePath", + "NameYAML": "logFilePath", + "NameGoAPI": "LogFilePath", + "StringSchema": "Path to file", + "StringSchemaHTML": "Path to file", + "StringDefault": "/var/log/calico/felix.log", + "ParsedDefault": "/var/log/calico/felix.log", + "ParsedDefaultJSON": "\"/var/log/calico/felix.log\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "/var/log/calico/felix.log", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "The full path to the Felix log. Set to none to disable file logging.", + "DescriptionHTML": "

The full path to the Felix log. Set to none to disable file logging.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Process: Logging", + "GroupWithSortPrefix": "00 Process: Logging", + "NameConfigFile": "LogPrefix", + "NameEnvVar": "FELIX_LogPrefix", + "NameYAML": "logPrefix", + "NameGoAPI": "LogPrefix", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "calico-packet", + "ParsedDefault": "calico-packet", + "ParsedDefaultJSON": "\"calico-packet\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String matching the regular expression `^([a-zA-Z0-9%: /_-])*$`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String matching the regular expression ^([a-zA-Z0-9%: /_-])*$.", + "YAMLDefault": "calico-packet", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The log prefix that Felix uses when rendering LOG rules. It is possible to use the following specifiers\nto include extra information in the log prefix.\n- %t: Tier name.\n- %k: Kind (short names).\n- %n: Policy or profile name.\n- %p: Policy or profile name (namespace/name for namespaced kinds or just name for non namespaced kinds).\nCalico includes \": \" characters at the end of the generated log prefix.\nNote that iptables shows up to 29 characters for the log prefix and nftables up to 127 characters. Extra characters are truncated.", + "DescriptionHTML": "

The log prefix that Felix uses when rendering LOG rules. It is possible to use the following specifiers\nto include extra information in the log prefix.\n- %t: Tier name.\n- %k: Kind (short names).\n- %n: Policy or profile name.\n- %p: Policy or profile name (namespace/name for namespaced kinds or just name for non namespaced kinds).\nCalico includes \": \" characters at the end of the generated log prefix.\nNote that iptables shows up to 29 characters for the log prefix and nftables up to 127 characters. Extra characters are truncated.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Process: Logging", + "GroupWithSortPrefix": "00 Process: Logging", + "NameConfigFile": "LogSeverityFile", + "NameEnvVar": "FELIX_LogSeverityFile", + "NameYAML": "logSeverityFile", + "NameGoAPI": "LogSeverityFile", + "StringSchema": "One of: `DEBUG`, `ERROR`, `FATAL`, `INFO`, `TRACE`, `WARNING`", + "StringSchemaHTML": "One of: DEBUG, ERROR, FATAL, INFO, TRACE, WARNING", + "StringDefault": "INFO", + "ParsedDefault": "INFO", + "ParsedDefaultJSON": "\"INFO\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Debug`, `Error`, `Fatal`, `Info`, `Trace`, `Warning`.", + "YAMLEnumValues": [ + "Debug", + "Error", + "Fatal", + "Info", + "Trace", + "Warning" + ], + "YAMLSchemaHTML": "One of: Debug, Error, Fatal, Info, Trace, Warning.", + "YAMLDefault": "Info", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The log severity above which logs are sent to the log file.", + "DescriptionHTML": "

The log severity above which logs are sent to the log file.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Process: Logging", + "GroupWithSortPrefix": "00 Process: Logging", + "NameConfigFile": "LogSeverityScreen", + "NameEnvVar": "FELIX_LogSeverityScreen", + "NameYAML": "logSeverityScreen", + "NameGoAPI": "LogSeverityScreen", + "StringSchema": "One of: `DEBUG`, `ERROR`, `FATAL`, `INFO`, `TRACE`, `WARNING`", + "StringSchemaHTML": "One of: DEBUG, ERROR, FATAL, INFO, TRACE, WARNING", + "StringDefault": "INFO", + "ParsedDefault": "INFO", + "ParsedDefaultJSON": "\"INFO\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Debug`, `Error`, `Fatal`, `Info`, `Trace`, `Warning`.", + "YAMLEnumValues": [ + "Debug", + "Error", + "Fatal", + "Info", + "Trace", + "Warning" + ], + "YAMLSchemaHTML": "One of: Debug, Error, Fatal, Info, Trace, Warning.", + "YAMLDefault": "Info", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The log severity above which logs are sent to the stdout.", + "DescriptionHTML": "

The log severity above which logs are sent to the stdout.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Process: Logging", + "GroupWithSortPrefix": "00 Process: Logging", + "NameConfigFile": "LogSeveritySys", + "NameEnvVar": "FELIX_LogSeveritySys", + "NameYAML": "logSeveritySys", + "NameGoAPI": "LogSeveritySys", + "StringSchema": "One of: `DEBUG`, `ERROR`, `FATAL`, `INFO`, `TRACE`, `WARNING`", + "StringSchemaHTML": "One of: DEBUG, ERROR, FATAL, INFO, TRACE, WARNING", + "StringDefault": "INFO", + "ParsedDefault": "INFO", + "ParsedDefaultJSON": "\"INFO\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Debug`, `Error`, `Fatal`, `Info`, `Trace`, `Warning`.", + "YAMLEnumValues": [ + "Debug", + "Error", + "Fatal", + "Info", + "Trace", + "Warning" + ], + "YAMLSchemaHTML": "One of: Debug, Error, Fatal, Info, Trace, Warning.", + "YAMLDefault": "Info", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The log severity above which logs are sent to the syslog. Set to None for no logging to syslog.", + "DescriptionHTML": "

The log severity above which logs are sent to the syslog. Set to None for no logging to syslog.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Process: Prometheus metrics", + "Fields": [ + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusGoMetricsEnabled", + "NameEnvVar": "FELIX_PrometheusGoMetricsEnabled", + "NameYAML": "prometheusGoMetricsEnabled", + "NameGoAPI": "PrometheusGoMetricsEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Disables Go runtime metrics collection, which the Prometheus client does by default, when\nset to false. This reduces the number of metrics reported, reducing Prometheus load.", + "DescriptionHTML": "

Disables Go runtime metrics collection, which the Prometheus client does by default, when\nset to false. This reduces the number of metrics reported, reducing Prometheus load.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusMetricsCAFile", + "NameEnvVar": "FELIX_PrometheusMetricsCAFile", + "NameYAML": "prometheusMetricsCAFile", + "NameGoAPI": "PrometheusMetricsCAFile", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Defines the absolute path to the TLS CA certificate file used for securing the /metrics endpoint.\nThis certificate must be valid and accessible by the calico-node process.", + "DescriptionHTML": "

Defines the absolute path to the TLS CA certificate file used for securing the /metrics endpoint.\nThis certificate must be valid and accessible by the calico-node process.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusMetricsCertFile", + "NameEnvVar": "FELIX_PrometheusMetricsCertFile", + "NameYAML": "prometheusMetricsCertFile", + "NameGoAPI": "PrometheusMetricsCertFile", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Defines the absolute path to the TLS certificate file used for securing the /metrics endpoint.\nThis certificate must be valid and accessible by the calico-node process.", + "DescriptionHTML": "

Defines the absolute path to the TLS certificate file used for securing the /metrics endpoint.\nThis certificate must be valid and accessible by the calico-node process.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusMetricsClientAuth", + "NameEnvVar": "FELIX_PrometheusMetricsClientAuth", + "NameYAML": "prometheusMetricsClientAuth", + "NameGoAPI": "PrometheusMetricsClientAuth", + "StringSchema": "One of: `NoClientCert`, `RequireAndVerifyClientCert`, `RequireAnyClientCert`, `VerifyClientCertIfGiven`", + "StringSchemaHTML": "One of: NoClientCert, RequireAndVerifyClientCert, RequireAnyClientCert, VerifyClientCertIfGiven", + "StringDefault": "RequireAndVerifyClientCert", + "ParsedDefault": "RequireAndVerifyClientCert", + "ParsedDefaultJSON": "\"RequireAndVerifyClientCert\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "RequireAndVerifyClientCert", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies the client authentication type for the /metrics endpoint.\nThis determines how the server validates client certificates. Default is \"RequireAndVerifyClientCert\".", + "DescriptionHTML": "

Specifies the client authentication type for the /metrics endpoint.\nThis determines how the server validates client certificates. Default is \"RequireAndVerifyClientCert\".

", + "UserEditable": true, + "GoType": "*v3.PrometheusMetricsClientAuthType", + "OpenSourceOnly": false + }, + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusMetricsEnabled", + "NameEnvVar": "FELIX_PrometheusMetricsEnabled", + "NameYAML": "prometheusMetricsEnabled", + "NameGoAPI": "PrometheusMetricsEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Enables the Prometheus metrics server in Felix if set to true.", + "DescriptionHTML": "

Enables the Prometheus metrics server in Felix if set to true.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusMetricsHost", + "NameEnvVar": "FELIX_PrometheusMetricsHost", + "NameYAML": "prometheusMetricsHost", + "NameGoAPI": "PrometheusMetricsHost", + "StringSchema": "String matching regex `^[a-zA-Z0-9:._+-]{1,64}$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9:._+-]{1,64}$", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The host that the Prometheus metrics server should bind to.", + "DescriptionHTML": "

The host that the Prometheus metrics server should bind to.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusMetricsKeyFile", + "NameEnvVar": "FELIX_PrometheusMetricsKeyFile", + "NameYAML": "prometheusMetricsKeyFile", + "NameGoAPI": "PrometheusMetricsKeyFile", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Defines the absolute path to the private key file corresponding to the TLS certificate\nused for securing the /metrics endpoint. The private key must be valid and accessible by the calico-node process.", + "DescriptionHTML": "

Defines the absolute path to the private key file corresponding to the TLS certificate\nused for securing the /metrics endpoint. The private key must be valid and accessible by the calico-node process.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusMetricsPort", + "NameEnvVar": "FELIX_PrometheusMetricsPort", + "NameYAML": "prometheusMetricsPort", + "NameGoAPI": "PrometheusMetricsPort", + "StringSchema": "Integer: [0,65535]", + "StringSchemaHTML": "Integer: [0,65535]", + "StringDefault": "9091", + "ParsedDefault": "9091", + "ParsedDefaultJSON": "9091", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [0,65535]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [0,65535]", + "YAMLDefault": "9091", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The TCP port that the Prometheus metrics server should bind to.", + "DescriptionHTML": "

The TCP port that the Prometheus metrics server should bind to.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusProcessMetricsEnabled", + "NameEnvVar": "FELIX_PrometheusProcessMetricsEnabled", + "NameYAML": "prometheusProcessMetricsEnabled", + "NameGoAPI": "PrometheusProcessMetricsEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Disables process metrics collection, which the Prometheus client does by default, when\nset to false. This reduces the number of metrics reported, reducing Prometheus load.", + "DescriptionHTML": "

Disables process metrics collection, which the Prometheus client does by default, when\nset to false. This reduces the number of metrics reported, reducing Prometheus load.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Process: Prometheus metrics", + "GroupWithSortPrefix": "00 Process: Prometheus metrics", + "NameConfigFile": "PrometheusWireGuardMetricsEnabled", + "NameEnvVar": "FELIX_PrometheusWireGuardMetricsEnabled", + "NameYAML": "prometheusWireGuardMetricsEnabled", + "NameGoAPI": "PrometheusWireGuardMetricsEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Disables wireguard metrics collection, which the Prometheus client does by default, when\nset to false. This reduces the number of metrics reported, reducing Prometheus load.", + "DescriptionHTML": "

Disables wireguard metrics collection, which the Prometheus client does by default, when\nset to false. This reduces the number of metrics reported, reducing Prometheus load.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Dataplane: Common", + "Fields": [ + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "AllowIPIPPacketsFromWorkloads", + "NameEnvVar": "FELIX_AllowIPIPPacketsFromWorkloads", + "NameYAML": "allowIPIPPacketsFromWorkloads", + "NameGoAPI": "AllowIPIPPacketsFromWorkloads", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix will add a rule to drop IPIP encapsulated traffic\nfrom workloads.", + "DescriptionHTML": "

Controls whether Felix will add a rule to drop IPIP encapsulated traffic\nfrom workloads.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "AllowVXLANPacketsFromWorkloads", + "NameEnvVar": "FELIX_AllowVXLANPacketsFromWorkloads", + "NameYAML": "allowVXLANPacketsFromWorkloads", + "NameGoAPI": "AllowVXLANPacketsFromWorkloads", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix will add a rule to drop VXLAN encapsulated traffic\nfrom workloads.", + "DescriptionHTML": "

Controls whether Felix will add a rule to drop VXLAN encapsulated traffic\nfrom workloads.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "CgroupV2Path", + "NameEnvVar": "FELIX_CgroupV2Path", + "NameYAML": "cgroupV2Path", + "NameGoAPI": "CgroupV2Path", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Overrides the default location where to find the cgroup hierarchy.", + "DescriptionHTML": "

Overrides the default location where to find the cgroup hierarchy.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "ChainInsertMode", + "NameEnvVar": "FELIX_ChainInsertMode", + "NameYAML": "chainInsertMode", + "NameGoAPI": "ChainInsertMode", + "StringSchema": "One of: `append`, `insert`", + "StringSchemaHTML": "One of: append, insert", + "StringDefault": "insert", + "ParsedDefault": "insert", + "ParsedDefaultJSON": "\"insert\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Append`, `Insert`.", + "YAMLEnumValues": [ + "Append", + "Insert" + ], + "YAMLSchemaHTML": "One of: Append, Insert.", + "YAMLDefault": "Insert", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix hooks the kernel's top-level iptables chains by inserting a rule\nat the top of the chain or by appending a rule at the bottom. insert is the safe default since it prevents\nCalico's rules from being bypassed. If you switch to append mode, be sure that the other rules in the chains\nsignal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed.", + "DescriptionHTML": "

Controls whether Felix hooks the kernel's top-level iptables chains by inserting a rule\nat the top of the chain or by appending a rule at the bottom. insert is the safe default since it prevents\nCalico's rules from being bypassed. If you switch to append mode, be sure that the other rules in the chains\nsignal acceptance by falling through to the Calico rules, otherwise the Calico policy will be bypassed.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "DataplaneDriver", + "NameEnvVar": "FELIX_DataplaneDriver", + "NameYAML": "dataplaneDriver", + "NameGoAPI": "DataplaneDriver", + "StringSchema": "Path to executable, which must exist. If not an absolute path, the directory containing this binary and the system path will be searched.", + "StringSchemaHTML": "Path to executable, which must exist. If not an absolute path, the directory containing this binary and the system path will be searched.", + "StringDefault": "calico-iptables-plugin", + "ParsedDefault": "calico-iptables-plugin", + "ParsedDefaultJSON": "\"calico-iptables-plugin\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "calico-iptables-plugin", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver\nis set to false.", + "DescriptionHTML": "

Filename of the external dataplane driver to use. Only used if UseInternalDataplaneDriver\nis set to false.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "DataplaneWatchdogTimeout", + "NameEnvVar": "FELIX_DataplaneWatchdogTimeout", + "NameYAML": "dataplaneWatchdogTimeout", + "NameGoAPI": "DataplaneWatchdogTimeout", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "90", + "ParsedDefault": "1m30s", + "ParsedDefaultJSON": "90000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1m30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The readiness/liveness timeout used for Felix's (internal) dataplane driver.\nDeprecated: replaced by the generic HealthTimeoutOverrides.", + "DescriptionHTML": "

The readiness/liveness timeout used for Felix's (internal) dataplane driver.\nDeprecated: replaced by the generic HealthTimeoutOverrides.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "DefaultEndpointToHostAction", + "NameEnvVar": "FELIX_DefaultEndpointToHostAction", + "NameYAML": "defaultEndpointToHostAction", + "NameGoAPI": "DefaultEndpointToHostAction", + "StringSchema": "One of: `ACCEPT`, `DROP`, `RETURN`", + "StringSchemaHTML": "One of: ACCEPT, DROP, RETURN", + "StringDefault": "DROP", + "ParsedDefault": "DROP", + "ParsedDefaultJSON": "\"DROP\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Accept`, `Drop`, `Return`.", + "YAMLEnumValues": [ + "Accept", + "Drop", + "Return" + ], + "YAMLSchemaHTML": "One of: Accept, Drop, Return.", + "YAMLDefault": "Drop", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls what happens to traffic that goes from a workload endpoint to the host\nitself (after the endpoint's egress policy is applied). By default, Calico blocks traffic from workload\nendpoints to the host itself with an iptables \"DROP\" action. If you want to allow some or all traffic from\nendpoint to host, set this parameter to RETURN or ACCEPT. Use RETURN if you have your own rules in the iptables\n\"INPUT\" chain; Calico will insert its rules at the top of that chain, then \"RETURN\" packets to the \"INPUT\" chain\nonce it has completed processing workload endpoint egress policy. Use ACCEPT to unconditionally accept packets\nfrom workloads after processing workload endpoint egress policy.", + "DescriptionHTML": "

Controls what happens to traffic that goes from a workload endpoint to the host\nitself (after the endpoint's egress policy is applied). By default, Calico blocks traffic from workload\nendpoints to the host itself with an iptables \"DROP\" action. If you want to allow some or all traffic from\nendpoint to host, set this parameter to RETURN or ACCEPT. Use RETURN if you have your own rules in the iptables\n\"INPUT\" chain; Calico will insert its rules at the top of that chain, then \"RETURN\" packets to the \"INPUT\" chain\nonce it has completed processing workload endpoint egress policy. Use ACCEPT to unconditionally accept packets\nfrom workloads after processing workload endpoint egress policy.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "DeviceRouteProtocol", + "NameEnvVar": "FELIX_DeviceRouteProtocol", + "NameYAML": "deviceRouteProtocol", + "NameGoAPI": "DeviceRouteProtocol", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "3", + "ParsedDefault": "3", + "ParsedDefaultJSON": "3", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "3", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the protocol to set on routes programmed by Felix. The protocol is an 8-bit label\nused to identify the owner of the route.", + "DescriptionHTML": "

Controls the protocol to set on routes programmed by Felix. The protocol is an 8-bit label\nused to identify the owner of the route.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "DeviceRouteSourceAddress", + "NameEnvVar": "FELIX_DeviceRouteSourceAddress", + "NameYAML": "deviceRouteSourceAddress", + "NameGoAPI": "DeviceRouteSourceAddress", + "StringSchema": "IPv4 address", + "StringSchemaHTML": "IPv4 address", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "net.IP", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "IPv4 address to set as the source hint for routes programmed by Felix. When not set\nthe source address for local traffic from host to workload will be determined by the kernel.", + "DescriptionHTML": "

IPv4 address to set as the source hint for routes programmed by Felix. When not set\nthe source address for local traffic from host to workload will be determined by the kernel.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "DeviceRouteSourceAddressIPv6", + "NameEnvVar": "FELIX_DeviceRouteSourceAddressIPv6", + "NameYAML": "deviceRouteSourceAddressIPv6", + "NameGoAPI": "DeviceRouteSourceAddressIPv6", + "StringSchema": "IPv6 address", + "StringSchemaHTML": "IPv6 address", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "net.IP", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "IPv6 address to set as the source hint for routes programmed by Felix. When not set\nthe source address for local traffic from host to workload will be determined by the kernel.", + "DescriptionHTML": "

IPv6 address to set as the source hint for routes programmed by Felix. When not set\nthe source address for local traffic from host to workload will be determined by the kernel.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "DisableConntrackInvalidCheck", + "NameEnvVar": "FELIX_DisableConntrackInvalidCheck", + "NameYAML": "disableConntrackInvalidCheck", + "NameGoAPI": "DisableConntrackInvalidCheck", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Disables the check for invalid connections in conntrack. While the conntrack\ninvalid check helps to detect malicious traffic, it can also cause issues with certain multi-NIC scenarios.", + "DescriptionHTML": "

Disables the check for invalid connections in conntrack. While the conntrack\ninvalid check helps to detect malicious traffic, it can also cause issues with certain multi-NIC scenarios.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "DropActionOverride", + "NameEnvVar": "FELIX_DropActionOverride", + "NameYAML": "dropActionOverride", + "NameGoAPI": "DropActionOverride", + "StringSchema": "One of: `ACCEPT`, `DROP`, `LOGandACCEPT`, `LOGandDROP`", + "StringSchemaHTML": "One of: ACCEPT, DROP, LOGandACCEPT, LOGandDROP", + "StringDefault": "DROP", + "ParsedDefault": "DROP", + "ParsedDefaultJSON": "\"DROP\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Accept`, `Drop`, `LogAndAccept`, `LogAndDrop`.", + "YAMLEnumValues": [ + "Accept", + "Drop", + "LogAndAccept", + "LogAndDrop" + ], + "YAMLSchemaHTML": "One of: Accept, Drop, LogAndAccept, LogAndDrop.", + "YAMLDefault": "Drop", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Overrides the Drop action in Felix, optionally changing the behavior to Accept, and optionally adding Log.\nPossible values are Drop, LogAndDrop, Accept, LogAndAccept.", + "DescriptionHTML": "

Overrides the Drop action in Felix, optionally changing the behavior to Accept, and optionally adding Log.\nPossible values are Drop, LogAndDrop, Accept, LogAndAccept.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "EndpointStatusPathPrefix", + "NameEnvVar": "FELIX_EndpointStatusPathPrefix", + "NameYAML": "endpointStatusPathPrefix", + "NameGoAPI": "EndpointStatusPathPrefix", + "StringSchema": "Path to file", + "StringSchemaHTML": "Path to file", + "StringDefault": "/var/run/calico", + "ParsedDefault": "/var/run/calico", + "ParsedDefaultJSON": "\"/var/run/calico\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "/var/run/calico", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The path to the directory where endpoint status will be written. Endpoint status\nfile reporting is disabled if field is left empty.\n\nChosen directory should match the directory used by the CNI plugin for PodStartupDelay.", + "DescriptionHTML": "

The path to the directory where endpoint status will be written. Endpoint status\nfile reporting is disabled if field is left empty.

\n

Chosen directory should match the directory used by the CNI plugin for PodStartupDelay.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "ExternalNodesCIDRList", + "NameEnvVar": "FELIX_ExternalNodesCIDRList", + "NameYAML": "externalNodesList", + "NameGoAPI": "ExternalNodesCIDRList", + "StringSchema": "Comma-delimited list of CIDRs", + "StringSchemaHTML": "Comma-delimited list of CIDRs", + "StringDefault": "", + "ParsedDefault": "[]", + "ParsedDefaultJSON": "null", + "ParsedType": "[]string", + "YAMLType": "array", + "YAMLSchema": "List of strings: `[\"\", ...]`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of strings: [\"<string>\", ...].", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "A list of CIDR's of external, non-Calico nodes from which VXLAN/IPIP overlay traffic\nwill be allowed. By default, external tunneled traffic is blocked to reduce attack surface.", + "DescriptionHTML": "

A list of CIDR's of external, non-Calico nodes from which VXLAN/IPIP overlay traffic\nwill be allowed. By default, external tunneled traffic is blocked to reduce attack surface.

", + "UserEditable": true, + "GoType": "*[]string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "FailsafeInboundHostPorts", + "NameEnvVar": "FELIX_FailsafeInboundHostPorts", + "NameYAML": "failsafeInboundHostPorts", + "NameGoAPI": "FailsafeInboundHostPorts", + "StringSchema": "Comma-delimited list of numeric ports with optional protocol and CIDR:`(tcp|udp)::`, `(tcp|udp):` or ``. IPv6 CIDRs must be enclosed in square brackets.", + "StringSchemaHTML": "Comma-delimited list of numeric ports with optional protocol and CIDR:(tcp|udp):<cidr>:<port>, (tcp|udp):<port> or <port>. IPv6 CIDRs must be enclosed in square brackets.", + "StringDefault": "tcp:22,udp:68,tcp:179,tcp:2379,tcp:2380,tcp:5473,tcp:6443,tcp:6666,tcp:6667", + "ParsedDefault": "[{tcp 22 } {udp 68 } {tcp 179 } {tcp 2379 } {tcp 2380 } {tcp 5473 } {tcp 6443 } {tcp 6666 } {tcp 6667 }]", + "ParsedDefaultJSON": "[{\"protocol\":\"tcp\",\"port\":22},{\"protocol\":\"udp\",\"port\":68},{\"protocol\":\"tcp\",\"port\":179},{\"protocol\":\"tcp\",\"port\":2379},{\"protocol\":\"tcp\",\"port\":2380},{\"protocol\":\"tcp\",\"port\":5473},{\"protocol\":\"tcp\",\"port\":6443},{\"protocol\":\"tcp\",\"port\":6666},{\"protocol\":\"tcp\",\"port\":6667}]", + "ParsedType": "[]v3.ProtoPort", + "YAMLType": "array", + "YAMLSchema": "List of protocol/port objects with optional CIDR match: `[{protocol: \"TCP|UDP\", port: , net: \"\"}, ...]`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of protocol/port objects with optional CIDR match: [{protocol: \"TCP|UDP\", port: <port>, net: \"<cidr>\"}, ...].", + "YAMLDefault": "[{\"protocol\":\"tcp\",\"port\":22},{\"protocol\":\"udp\",\"port\":68},{\"protocol\":\"tcp\",\"port\":179},{\"protocol\":\"tcp\",\"port\":2379},{\"protocol\":\"tcp\",\"port\":2380},{\"protocol\":\"tcp\",\"port\":5473},{\"protocol\":\"tcp\",\"port\":6443},{\"protocol\":\"tcp\",\"port\":6666},{\"protocol\":\"tcp\",\"port\":6667}]", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "A list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will\nallow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally\ncutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified,\nit defaults to \"tcp\". If a CIDR is not specified, it will allow traffic from all addresses. To disable all inbound host ports,\nuse the value \"[]\". The default value allows ssh access, DHCP, BGP, etcd and the Kubernetes API.", + "DescriptionHTML": "

A list of ProtoPort struct objects including UDP/TCP/SCTP ports and CIDRs that Felix will\nallow incoming traffic to host endpoints on irrespective of the security policy. This is useful to avoid accidentally\ncutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified,\nit defaults to \"tcp\". If a CIDR is not specified, it will allow traffic from all addresses. To disable all inbound host ports,\nuse the value \"[]\". The default value allows ssh access, DHCP, BGP, etcd and the Kubernetes API.

", + "UserEditable": true, + "GoType": "*[]v3.ProtoPort", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "FailsafeOutboundHostPorts", + "NameEnvVar": "FELIX_FailsafeOutboundHostPorts", + "NameYAML": "failsafeOutboundHostPorts", + "NameGoAPI": "FailsafeOutboundHostPorts", + "StringSchema": "Comma-delimited list of numeric ports with optional protocol and CIDR:`(tcp|udp)::`, `(tcp|udp):` or ``. IPv6 CIDRs must be enclosed in square brackets.", + "StringSchemaHTML": "Comma-delimited list of numeric ports with optional protocol and CIDR:(tcp|udp):<cidr>:<port>, (tcp|udp):<port> or <port>. IPv6 CIDRs must be enclosed in square brackets.", + "StringDefault": "udp:53,udp:67,tcp:179,tcp:2379,tcp:2380,tcp:5473,tcp:6443,tcp:6666,tcp:6667", + "ParsedDefault": "[{udp 53 } {udp 67 } {tcp 179 } {tcp 2379 } {tcp 2380 } {tcp 5473 } {tcp 6443 } {tcp 6666 } {tcp 6667 }]", + "ParsedDefaultJSON": "[{\"protocol\":\"udp\",\"port\":53},{\"protocol\":\"udp\",\"port\":67},{\"protocol\":\"tcp\",\"port\":179},{\"protocol\":\"tcp\",\"port\":2379},{\"protocol\":\"tcp\",\"port\":2380},{\"protocol\":\"tcp\",\"port\":5473},{\"protocol\":\"tcp\",\"port\":6443},{\"protocol\":\"tcp\",\"port\":6666},{\"protocol\":\"tcp\",\"port\":6667}]", + "ParsedType": "[]v3.ProtoPort", + "YAMLType": "array", + "YAMLSchema": "List of protocol/port objects with optional CIDR match: `[{protocol: \"TCP|UDP\", port: , net: \"\"}, ...]`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of protocol/port objects with optional CIDR match: [{protocol: \"TCP|UDP\", port: <port>, net: \"<cidr>\"}, ...].", + "YAMLDefault": "[{\"protocol\":\"udp\",\"port\":53},{\"protocol\":\"udp\",\"port\":67},{\"protocol\":\"tcp\",\"port\":179},{\"protocol\":\"tcp\",\"port\":2379},{\"protocol\":\"tcp\",\"port\":2380},{\"protocol\":\"tcp\",\"port\":5473},{\"protocol\":\"tcp\",\"port\":6443},{\"protocol\":\"tcp\",\"port\":6666},{\"protocol\":\"tcp\",\"port\":6667}]", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "A list of PortProto struct objects including UDP/TCP/SCTP ports and CIDRs that Felix\nwill allow outgoing traffic from host endpoints to irrespective of the security policy. This is useful to avoid accidentally\ncutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults\nto \"tcp\". If a CIDR is not specified, it will allow traffic from all addresses. To disable all outbound host ports,\nuse the value \"[]\". The default value opens etcd's standard ports to ensure that Felix does not get cut off from etcd\nas well as allowing DHCP, DNS, BGP and the Kubernetes API.", + "DescriptionHTML": "

A list of PortProto struct objects including UDP/TCP/SCTP ports and CIDRs that Felix\nwill allow outgoing traffic from host endpoints to irrespective of the security policy. This is useful to avoid accidentally\ncutting off a host with incorrect configuration. For backwards compatibility, if the protocol is not specified, it defaults\nto \"tcp\". If a CIDR is not specified, it will allow traffic from all addresses. To disable all outbound host ports,\nuse the value \"[]\". The default value opens etcd's standard ports to ensure that Felix does not get cut off from etcd\nas well as allowing DHCP, DNS, BGP and the Kubernetes API.

", + "UserEditable": true, + "GoType": "*[]v3.ProtoPort", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "FloatingIPs", + "NameEnvVar": "FELIX_FloatingIPs", + "NameYAML": "floatingIPs", + "NameGoAPI": "FloatingIPs", + "StringSchema": "One of: `Disabled`, `Enabled`", + "StringSchemaHTML": "One of: Disabled, Enabled", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Disabled\"`, `\"Enabled\"`.", + "YAMLEnumValues": [ + "Disabled", + "Enabled" + ], + "YAMLSchemaHTML": "One of: \"Disabled\", \"Enabled\".", + "YAMLDefault": "Disabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures whether or not Felix will program non-OpenStack floating IP addresses. (OpenStack-derived\nfloating IPs are always programmed, regardless of this setting.)", + "DescriptionHTML": "

Configures whether or not Felix will program non-OpenStack floating IP addresses. (OpenStack-derived\nfloating IPs are always programmed, regardless of this setting.)

", + "UserEditable": true, + "GoType": "*v3.FloatingIPType", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "IPForwarding", + "NameEnvVar": "FELIX_IPForwarding", + "NameYAML": "ipForwarding", + "NameGoAPI": "IPForwarding", + "StringSchema": "One of: `Disabled`, `Enabled`", + "StringSchemaHTML": "One of: Disabled, Enabled", + "StringDefault": "Enabled", + "ParsedDefault": "Enabled", + "ParsedDefaultJSON": "\"Enabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Disabled\"`, `\"Enabled\"`.", + "YAMLEnumValues": [ + "Disabled", + "Enabled" + ], + "YAMLSchemaHTML": "One of: \"Disabled\", \"Enabled\".", + "YAMLDefault": "Enabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required\nwhen using Calico for workload networking. This should be disabled only on hosts where Calico is used solely for\nhost protection. In BPF mode, due to a kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF\nmust be disabled.", + "DescriptionHTML": "

Controls whether Felix sets the host sysctls to enable IP forwarding. IP forwarding is required\nwhen using Calico for workload networking. This should be disabled only on hosts where Calico is used solely for\nhost protection. In BPF mode, due to a kernel interaction, either IPForwarding must be enabled or BPFEnforceRPF\nmust be disabled.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "IPv4ElevatedRoutePriority", + "NameEnvVar": "FELIX_IPv4ElevatedRoutePriority", + "NameYAML": "ipv4ElevatedRoutePriority", + "NameGoAPI": "IPv4ElevatedRoutePriority", + "StringSchema": "Integer: [1,2147483646]", + "StringSchemaHTML": "Integer: [1,2147483646]", + "StringDefault": "512", + "ParsedDefault": "512", + "ParsedDefaultJSON": "512", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [1,2147483646]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [1,2147483646]", + "YAMLDefault": "512", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Route Priority value for an elevated priority Calico-programmed IPv4 route. Note, higher\nvalues mean lower priority. Elevated priority is used during VM live migration, and for\noptimal behaviour IPv4ElevatedRoutePriority must be less than IPv4NormalRoutePriority.", + "DescriptionHTML": "

Route Priority value for an elevated priority Calico-programmed IPv4 route. Note, higher\nvalues mean lower priority. Elevated priority is used during VM live migration, and for\noptimal behaviour IPv4ElevatedRoutePriority must be less than IPv4NormalRoutePriority.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "IPv4NormalRoutePriority", + "NameEnvVar": "FELIX_IPv4NormalRoutePriority", + "NameYAML": "ipv4NormalRoutePriority", + "NameGoAPI": "IPv4NormalRoutePriority", + "StringSchema": "Integer: [1,2147483646]", + "StringSchemaHTML": "Integer: [1,2147483646]", + "StringDefault": "1024", + "ParsedDefault": "1024", + "ParsedDefaultJSON": "1024", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [1,2147483646]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [1,2147483646]", + "YAMLDefault": "1024", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Route Priority value for a normal priority Calico-programmed IPv4 route. Note, higher\nvalues mean lower priority.", + "DescriptionHTML": "

Route Priority value for a normal priority Calico-programmed IPv4 route. Note, higher\nvalues mean lower priority.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "IPv6ElevatedRoutePriority", + "NameEnvVar": "FELIX_IPv6ElevatedRoutePriority", + "NameYAML": "ipv6ElevatedRoutePriority", + "NameGoAPI": "IPv6ElevatedRoutePriority", + "StringSchema": "Integer: [1,2147483646]", + "StringSchemaHTML": "Integer: [1,2147483646]", + "StringDefault": "512", + "ParsedDefault": "512", + "ParsedDefaultJSON": "512", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [1,2147483646]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [1,2147483646]", + "YAMLDefault": "512", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Route Priority value for an elevated priority Calico-programmed IPv6 route. Note, higher\nvalues mean lower priority. Elevated priority is used during VM live migration, and for\noptimal behaviour IPv6ElevatedRoutePriority must be less than IPv6NormalRoutePriority.", + "DescriptionHTML": "

Route Priority value for an elevated priority Calico-programmed IPv6 route. Note, higher\nvalues mean lower priority. Elevated priority is used during VM live migration, and for\noptimal behaviour IPv6ElevatedRoutePriority must be less than IPv6NormalRoutePriority.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "IPv6NormalRoutePriority", + "NameEnvVar": "FELIX_IPv6NormalRoutePriority", + "NameYAML": "ipv6NormalRoutePriority", + "NameGoAPI": "IPv6NormalRoutePriority", + "StringSchema": "Integer: [1,2147483646]", + "StringSchemaHTML": "Integer: [1,2147483646]", + "StringDefault": "1024", + "ParsedDefault": "1024", + "ParsedDefaultJSON": "1024", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [1,2147483646]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [1,2147483646]", + "YAMLDefault": "1024", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Route Priority value for a normal priority Calico-programmed IPv6 route. Note, higher\nvalues mean lower priority.", + "DescriptionHTML": "

Route Priority value for a normal priority Calico-programmed IPv6 route. Note, higher\nvalues mean lower priority.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "InterfaceExclude", + "NameEnvVar": "FELIX_InterfaceExclude", + "NameYAML": "interfaceExclude", + "NameGoAPI": "InterfaceExclude", + "StringSchema": "Comma-delimited list of Linux interface names/regex patterns. Regex patterns must start/end with `/`.", + "StringSchemaHTML": "Comma-delimited list of Linux interface names/regex patterns. Regex patterns must start/end with /.", + "StringDefault": "kube-ipvs0", + "ParsedDefault": "[^kube-ipvs0$]", + "ParsedDefaultJSON": "[\"^kube-ipvs0$\"]", + "ParsedType": "[]*regexp.Regexp", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "kube-ipvs0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A comma-separated list of interface names that should be excluded when Felix is resolving\nhost endpoints. The default value ensures that Felix ignores Kubernetes' internal `kube-ipvs0` device. If you\nwant to exclude multiple interface names using a single value, the list supports regular expressions. For\nregular expressions you must wrap the value with `/`. For example having values `/^kube/,veth1` will exclude\nall interfaces that begin with `kube` and also the interface `veth1`.", + "DescriptionHTML": "

A comma-separated list of interface names that should be excluded when Felix is resolving\nhost endpoints. The default value ensures that Felix ignores Kubernetes' internal kube-ipvs0 device. If you\nwant to exclude multiple interface names using a single value, the list supports regular expressions. For\nregular expressions you must wrap the value with /. For example having values /^kube/,veth1 will exclude\nall interfaces that begin with kube and also the interface veth1.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "InterfacePrefix", + "NameEnvVar": "FELIX_InterfacePrefix", + "NameYAML": "interfacePrefix", + "NameGoAPI": "InterfacePrefix", + "StringSchema": "String matching regex `^[a-zA-Z0-9_-]{1,15}(,[a-zA-Z0-9_-]{1,15})*$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9_-]{1,15}(,[a-zA-Z0-9_-]{1,15})*$", + "StringDefault": "cali", + "ParsedDefault": "cali", + "ParsedDefaultJSON": "\"cali\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "cali", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "The interface name prefix that identifies workload endpoints and so distinguishes\nthem from host endpoint interfaces. Note: in environments other than bare metal, the orchestrators\nconfigure this appropriately. For example our Kubernetes and Docker integrations set the 'cali' value,\nand our OpenStack integration sets the 'tap' value.", + "DescriptionHTML": "

The interface name prefix that identifies workload endpoints and so distinguishes\nthem from host endpoint interfaces. Note: in environments other than bare metal, the orchestrators\nconfigure this appropriately. For example our Kubernetes and Docker integrations set the 'cali' value,\nand our OpenStack integration sets the 'tap' value.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "InterfaceRefreshInterval", + "NameEnvVar": "FELIX_InterfaceRefreshInterval", + "NameYAML": "interfaceRefreshInterval", + "NameGoAPI": "InterfaceRefreshInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "90", + "ParsedDefault": "1m30s", + "ParsedDefaultJSON": "90000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1m30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The period at which Felix rescans local interfaces to verify their state.\nThe rescan can be disabled by setting the interval to 0.", + "DescriptionHTML": "

The period at which Felix rescans local interfaces to verify their state.\nThe rescan can be disabled by setting the interval to 0.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "Ipv6Support", + "NameEnvVar": "FELIX_Ipv6Support", + "NameYAML": "ipv6Support", + "NameGoAPI": "IPv6Support", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix enables support for IPv6 (if supported by the in-use dataplane).", + "DescriptionHTML": "

Controls whether Felix enables support for IPv6 (if supported by the in-use dataplane).

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "IstioAmbientMode", + "NameEnvVar": "FELIX_IstioAmbientMode", + "NameYAML": "istioAmbientMode", + "NameGoAPI": "IstioAmbientMode", + "StringSchema": "One of: `Disabled`, `Enabled`", + "StringSchemaHTML": "One of: Disabled, Enabled", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Disabled\"`, `\"Enabled\"`.", + "YAMLEnumValues": [ + "Disabled", + "Enabled" + ], + "YAMLSchemaHTML": "One of: \"Disabled\", \"Enabled\".", + "YAMLDefault": "Disabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures Felix to work together with Tigera's Istio distribution.", + "DescriptionHTML": "

Configures Felix to work together with Tigera's Istio distribution.

", + "UserEditable": true, + "GoType": "*v3.IstioAmbientMode", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "IstioDSCPMark", + "NameEnvVar": "FELIX_IstioDSCPMark", + "NameYAML": "istioDSCPMark", + "NameGoAPI": "IstioDSCPMark", + "StringSchema": "Numeric value: An integer from 0 to 63, representing the 6-bit DSCP code directly; Named value: A case-insensitive string corresponding to a standardized DSCP name (e.g., \"CS0\", \"AF11\", \"AF21\", \"EF\", etc.) as defined in the IANA registry for Differentiated Services Field Codepoints.", + "StringSchemaHTML": "Numeric value: An integer from 0 to 63, representing the 6-bit DSCP code directly; Named value: A case-insensitive string corresponding to a standardized DSCP name (e.g., \"CS0\", \"AF11\", \"AF21\", \"EF\", etc.) as defined in the IANA registry for Differentiated Services Field Codepoints.", + "StringDefault": "23", + "ParsedDefault": "23", + "ParsedDefaultJSON": "23", + "ParsedType": "numorstring.DSCP", + "YAMLType": "integer", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the value to use when directing traffic to Istio ZTunnel, when Istio is enabled. The mark is set only on\nSYN packets at the final hop to avoid interference with other protocols. This value is reserved by Calico and must not be used\nwith other Istio installation.", + "DescriptionHTML": "

Sets the value to use when directing traffic to Istio ZTunnel, when Istio is enabled. The mark is set only on\nSYN packets at the final hop to avoid interference with other protocols. This value is reserved by Calico and must not be used\nwith other Istio installation.

", + "UserEditable": true, + "GoType": "*numorstring.DSCP", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "KubeMasqueradeBit", + "NameEnvVar": "FELIX_KubeMasqueradeBit", + "NameYAML": "kubeMasqueradeBit", + "NameGoAPI": "KubeMasqueradeBit", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "14", + "ParsedDefault": "14", + "ParsedDefaultJSON": "14", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "14", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Should be set to the same value as --iptables-masquerade-bit of kube-proxy\nwhen TPROXY is used. The default is the same as kube-proxy default thus only needs a change\nif kube-proxy is using a non-standard setting. Must be within the range of 0-31.", + "DescriptionHTML": "

Should be set to the same value as --iptables-masquerade-bit of kube-proxy\nwhen TPROXY is used. The default is the same as kube-proxy default thus only needs a change\nif kube-proxy is using a non-standard setting. Must be within the range of 0-31.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "L7ObservabilityEnabled", + "NameEnvVar": "FELIX_L7ObservabilityEnabled", + "NameYAML": "l7ObservabilityEnabled", + "NameGoAPI": "L7ObservabilityEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Enables eBPF-based L7 HTTP and TLS observability.\nIt is dataplane-agnostic - works with eBPF, iptables, or nftables.\nRequires kernel 5.17+.", + "DescriptionHTML": "

Enables eBPF-based L7 HTTP and TLS observability.\nIt is dataplane-agnostic - works with eBPF, iptables, or nftables.\nRequires kernel 5.17+.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "LiveMigrationRouteConvergenceTime", + "NameEnvVar": "FELIX_LiveMigrationRouteConvergenceTime", + "NameYAML": "liveMigrationRouteConvergenceTime", + "NameGoAPI": "LiveMigrationRouteConvergenceTime", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "30", + "ParsedDefault": "30s", + "ParsedDefaultJSON": "30000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The time to keep elevated route priority after a\nVM live migration completes. This allows routes to converge across the cluster before\nreverting to normal priority.", + "DescriptionHTML": "

The time to keep elevated route priority after a\nVM live migration completes. This allows routes to converge across the cluster before\nreverting to normal priority.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "LocalSubnetL2Reachability", + "NameEnvVar": "FELIX_LocalSubnetL2Reachability", + "NameYAML": "localSubnetL2Reachability", + "NameGoAPI": "LocalSubnetL2Reachability", + "StringSchema": "One of: `Disabled`, `PodsAndLoadBalancers`", + "StringSchemaHTML": "One of: Disabled, PodsAndLoadBalancers", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Disabled\"`, `\"PodsAndLoadBalancers\"`.", + "YAMLEnumValues": [ + "Disabled", + "PodsAndLoadBalancers" + ], + "YAMLSchemaHTML": "One of: \"Disabled\", \"PodsAndLoadBalancers\".", + "YAMLDefault": "Disabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix automatically responds to\nARP (IPv4) and NDP (IPv6) requests on host interfaces for local pod IPs and\nselected LoadBalancer VIPs that fall within the same subnet as the host\ninterface. When set to PodsAndLoadBalancers, pods and LB VIPs on the host\nsubnet are reachable from the local L2 segment without BGP.", + "DescriptionHTML": "

Controls whether Felix automatically responds to\nARP (IPv4) and NDP (IPv6) requests on host interfaces for local pod IPs and\nselected LoadBalancer VIPs that fall within the same subnet as the host\ninterface. When set to PodsAndLoadBalancers, pods and LB VIPs on the host\nsubnet are reachable from the local L2 segment without BGP.

", + "UserEditable": true, + "GoType": "*v3.LocalSubnetL2ReachabilityMode", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "LocalSubnetL2ReachabilityRefreshInterval", + "NameEnvVar": "FELIX_LocalSubnetL2ReachabilityRefreshInterval", + "NameYAML": "localSubnetL2ReachabilityRefreshInterval", + "NameGoAPI": "LocalSubnetL2ReachabilityRefreshInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "120", + "ParsedDefault": "2m0s", + "ParsedDefaultJSON": "120000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "2m0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls how often Felix re-announces\n(gratuitous ARP / unsolicited NA) every IP it proxies ARP/NDP for when\nLocalSubnetL2Reachability is enabled, keeping neighbor caches and switch\nforwarding tables warm even when the set of proxied IPs is unchanged. Set to 0\nto disable periodic re-announcement, leaving only the one-shot announce when an\nIP is added.", + "DescriptionHTML": "

Controls how often Felix re-announces\n(gratuitous ARP / unsolicited NA) every IP it proxies ARP/NDP for when\nLocalSubnetL2Reachability is enabled, keeping neighbor caches and switch\nforwarding tables warm even when the set of proxied IPs is unchanged. Set to 0\nto disable periodic re-announcement, leaving only the one-shot announce when an\nIP is added.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "MTUIfacePattern", + "NameEnvVar": "FELIX_MTUIfacePattern", + "NameYAML": "mtuIfacePattern", + "NameGoAPI": "MTUIfacePattern", + "StringSchema": "Regular expression", + "StringSchemaHTML": "Regular expression", + "StringDefault": "^((en|wl|ww|sl|ib)[Pcopsvx].*|(eth|wlan|wwan).*)", + "ParsedDefault": "^((en|wl|ww|sl|ib)[Pcopsvx].*|(eth|wlan|wwan).*)", + "ParsedDefaultJSON": "\"^((en|wl|ww|sl|ib)[Pcopsvx].*|(eth|wlan|wwan).*)\"", + "ParsedType": "*regexp.Regexp", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "^((en|wl|ww|sl|ib)[Pcopsvx].*|(eth|wlan|wwan).*)", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A regular expression that controls which interfaces Felix should scan in order\nto calculate the host's MTU.\nThis should not match workload interfaces (usually named cali...).", + "DescriptionHTML": "

A regular expression that controls which interfaces Felix should scan in order\nto calculate the host's MTU.\nThis should not match workload interfaces (usually named cali...).

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "NATOutgoingAddress", + "NameEnvVar": "FELIX_NATOutgoingAddress", + "NameYAML": "natOutgoingAddress", + "NameGoAPI": "NATOutgoingAddress", + "StringSchema": "IPv4 address", + "StringSchemaHTML": "IPv4 address", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "net.IP", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies an address to use when performing source NAT for traffic in a natOutgoing pool that\nis leaving the network. By default the address used is an address on the interface the traffic is leaving on\n(i.e. it uses the iptables MASQUERADE target).", + "DescriptionHTML": "

Specifies an address to use when performing source NAT for traffic in a natOutgoing pool that\nis leaving the network. By default the address used is an address on the interface the traffic is leaving on\n(i.e. it uses the iptables MASQUERADE target).

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "NATOutgoingExclusions", + "NameEnvVar": "FELIX_NATOutgoingExclusions", + "NameYAML": "natOutgoingExclusions", + "NameGoAPI": "NATOutgoingExclusions", + "StringSchema": "One of: `IPPoolsAndHostIPs`, `IPPoolsOnly`", + "StringSchemaHTML": "One of: IPPoolsAndHostIPs, IPPoolsOnly", + "StringDefault": "IPPoolsOnly", + "ParsedDefault": "IPPoolsOnly", + "ParsedDefaultJSON": "\"IPPoolsOnly\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"IPPoolsAndHostIPs\"`, `\"IPPoolsOnly\"`.", + "YAMLEnumValues": [ + "IPPoolsAndHostIPs", + "IPPoolsOnly" + ], + "YAMLSchemaHTML": "One of: \"IPPoolsAndHostIPs\", \"IPPoolsOnly\".", + "YAMLDefault": "IPPoolsOnly", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "When a IP pool setting `natOutgoing` is true, packets sent from Calico networked containers in this IP pool to destinations will be masqueraded.\nConfigure which type of destinations is excluded from being masqueraded.\n- IPPoolsOnly: destinations outside of this IP pool will be masqueraded.\n- IPPoolsAndHostIPs: destinations outside of this IP pool and all hosts will be masqueraded.", + "DescriptionHTML": "

When a IP pool setting natOutgoing is true, packets sent from Calico networked containers in this IP pool to destinations will be masqueraded.\nConfigure which type of destinations is excluded from being masqueraded.\n- IPPoolsOnly: destinations outside of this IP pool will be masqueraded.\n- IPPoolsAndHostIPs: destinations outside of this IP pool and all hosts will be masqueraded.

", + "UserEditable": true, + "GoType": "*v3.NATOutgoingExclusionsType", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "NATPortRange", + "NameEnvVar": "FELIX_NATPortRange", + "NameYAML": "natPortRange", + "NameGoAPI": "NATPortRange", + "StringSchema": "Port range: either a single number in [0,65535] or a range of numbers `n:m`", + "StringSchemaHTML": "Port range: either a single number in [0,65535] or a range of numbers n:m", + "StringDefault": "", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "numorstring.Port", + "YAMLType": "integer or string", + "YAMLSchema": "Port range: either an integer in [0,65535] or a string, representing a range, in format `n:m`", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Port range: either an integer in [0,65535] or a string, representing a range, in format n:m", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies the range of ports that is used for port mapping when doing outgoing NAT. When unset the default behavior of the\nnetwork stack is used.", + "DescriptionHTML": "

Specifies the range of ports that is used for port mapping when doing outgoing NAT. When unset the default behavior of the\nnetwork stack is used.

", + "UserEditable": true, + "GoType": "*numorstring.Port", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "NFTablesDNSPolicyMode", + "NameEnvVar": "FELIX_NFTablesDNSPolicyMode", + "NameYAML": "nftablesDNSPolicyMode", + "NameGoAPI": "NFTablesDNSPolicyMode", + "StringSchema": "One of: `DelayDNSResponse`, `DelayDeniedPacket`, `NoDelay`", + "StringSchemaHTML": "One of: DelayDNSResponse, DelayDeniedPacket, NoDelay", + "StringDefault": "DelayDeniedPacket", + "ParsedDefault": "DelayDeniedPacket", + "ParsedDefaultJSON": "\"DelayDeniedPacket\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"DelayDNSResponse\"`, `\"DelayDeniedPacket\"`, `\"NoDelay\"`.", + "YAMLEnumValues": [ + "DelayDNSResponse", + "DelayDeniedPacket", + "NoDelay" + ], + "YAMLSchemaHTML": "One of: \"DelayDNSResponse\", \"DelayDeniedPacket\", \"NoDelay\".", + "YAMLDefault": "DelayDeniedPacket", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies how DNS policy programming will be handled for NFTables.\nDelayDeniedPacket - Felix delays any denied packet that traversed a policy that included egress domain matches,\nbut did not match. The packet is released after a fixed time, or after the destination IP address was programmed.\nDelayDNSResponse - Felix delays any DNS response until related IPSets are programmed. This introduces some\nlatency to all DNS packets (even when no IPSet programming is required), but it ensures policy hit statistics\nare accurate. This is the recommended setting when you are making use of staged policies or policy rule hit\nstatistics.\nNoDelay - Felix does not introduce any delay to the packets. DNS rules may not have been programmed by the time\nthe first packet traverses the policy rules. Client applications need to handle reconnection attempts if initial\nconnection attempts fail. This may be problematic for some applications or for very low DNS TTLs.", + "DescriptionHTML": "

Specifies how DNS policy programming will be handled for NFTables.\nDelayDeniedPacket - Felix delays any denied packet that traversed a policy that included egress domain matches,\nbut did not match. The packet is released after a fixed time, or after the destination IP address was programmed.\nDelayDNSResponse - Felix delays any DNS response until related IPSets are programmed. This introduces some\nlatency to all DNS packets (even when no IPSet programming is required), but it ensures policy hit statistics\nare accurate. This is the recommended setting when you are making use of staged policies or policy rule hit\nstatistics.\nNoDelay - Felix does not introduce any delay to the packets. DNS rules may not have been programmed by the time\nthe first packet traverses the policy rules. Client applications need to handle reconnection attempts if initial\nconnection attempts fail. This may be problematic for some applications or for very low DNS TTLs.

", + "UserEditable": true, + "GoType": "*v3.NFTablesDNSPolicyMode", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "NFTablesMode", + "NameEnvVar": "FELIX_NFTablesMode", + "NameYAML": "nftablesMode", + "NameGoAPI": "NFTablesMode", + "StringSchema": "One of: `Auto`, `Disabled`, `Enabled`", + "StringSchemaHTML": "One of: Auto, Disabled, Enabled", + "StringDefault": "Auto", + "ParsedDefault": "Auto", + "ParsedDefaultJSON": "\"Auto\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Auto\"`, `\"Disabled\"`, `\"Enabled\"`.", + "YAMLEnumValues": [ + "Auto", + "Disabled", + "Enabled" + ], + "YAMLSchemaHTML": "One of: \"Auto\", \"Disabled\", \"Enabled\".", + "YAMLDefault": "Auto", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures nftables support in Felix.", + "DescriptionHTML": "

Configures nftables support in Felix.

", + "UserEditable": true, + "GoType": "*v3.NFTablesMode", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "NetlinkTimeoutSecs", + "NameEnvVar": "FELIX_NetlinkTimeoutSecs", + "NameYAML": "netlinkTimeout", + "NameGoAPI": "NetlinkTimeout", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "10", + "ParsedDefault": "10s", + "ParsedDefaultJSON": "10000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "10s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The timeout when talking to the kernel over the netlink protocol, used for programming\nroutes, rules, and other kernel objects.", + "DescriptionHTML": "

The timeout when talking to the kernel over the netlink protocol, used for programming\nroutes, rules, and other kernel objects.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "NfNetlinkBufSize", + "NameEnvVar": "FELIX_NfNetlinkBufSize", + "NameYAML": "nfNetlinkBufSize", + "NameGoAPI": "NfNetlinkBufSize", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "65536", + "ParsedDefault": "65536", + "ParsedDefaultJSON": "65536", + "ParsedType": "int", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "65536", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the size of NFLOG messages that the kernel will try to send to Felix. NFLOG messages\nare used to report flow verdicts from the kernel. Warning: currently increasing the value may cause errors\ndue to a bug in the netlink library.", + "DescriptionHTML": "

Controls the size of NFLOG messages that the kernel will try to send to Felix. NFLOG messages\nare used to report flow verdicts from the kernel. Warning: currently increasing the value may cause errors\ndue to a bug in the netlink library.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "PolicyActivityLogsFileDirectory", + "NameEnvVar": "FELIX_PolicyActivityLogsFileDirectory", + "NameYAML": "policyActivityLogsFileDirectory", + "NameGoAPI": "PolicyActivityLogsFileDirectory", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "/var/log/calico/policy", + "ParsedDefault": "/var/log/calico/policy", + "ParsedDefaultJSON": "\"/var/log/calico/policy\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "/var/log/calico/policy", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the directory where policy activity log files are stored.", + "DescriptionHTML": "

Sets the directory where policy activity log files are stored.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "PolicyActivityLogsFileEnabled", + "NameEnvVar": "FELIX_PolicyActivityLogsFileEnabled", + "NameYAML": "policyActivityLogsFileEnabled", + "NameGoAPI": "PolicyActivityLogsFileEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls logging policy activity logs to a file. If false no policy activity logging to file will occur.", + "DescriptionHTML": "

Controls logging policy activity logs to a file. If false no policy activity logging to file will occur.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "PolicyActivityLogsFileMaxFileSizeMB", + "NameEnvVar": "FELIX_PolicyActivityLogsFileMaxFileSizeMB", + "NameYAML": "policyActivityLogsFileMaxFileSizeMB", + "NameGoAPI": "PolicyActivityLogsFileMaxFileSizeMB", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "100", + "ParsedDefault": "100", + "ParsedDefaultJSON": "100", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "100", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the max size in MB of policy activity log files before rotation.", + "DescriptionHTML": "

Sets the max size in MB of policy activity log files before rotation.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "PolicyActivityLogsFileMaxFiles", + "NameEnvVar": "FELIX_PolicyActivityLogsFileMaxFiles", + "NameYAML": "policyActivityLogsFileMaxFiles", + "NameGoAPI": "PolicyActivityLogsFileMaxFiles", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "5", + "ParsedDefault": "5", + "ParsedDefaultJSON": "5", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "5", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the number of policy activity log files to keep.", + "DescriptionHTML": "

Sets the number of policy activity log files to keep.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "PolicyActivityLogsFlushInterval", + "NameEnvVar": "FELIX_PolicyActivityLogsFlushInterval", + "NameYAML": "policyActivityLogsFlushInterval", + "NameGoAPI": "PolicyActivityLogsFlushInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "15", + "ParsedDefault": "15s", + "ParsedDefaultJSON": "15000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "15s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures the interval at which Felix exports policy activity logs.", + "DescriptionHTML": "

Configures the interval at which Felix exports policy activity logs.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "PolicyActivityRefreshInterval", + "NameEnvVar": "FELIX_PolicyActivityRefreshInterval", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "3600", + "ParsedDefault": "1h0m0s", + "ParsedDefaultJSON": "3600000000000", + "ParsedType": "time.Duration", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Controls how often Felix re-evaluates policies for\nlong-lived connections to keep policy activity timestamps current.", + "DescriptionHTML": "

Controls how often Felix re-evaluates policies for\nlong-lived connections to keep policy activity timestamps current.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "PolicySyncPathPrefix", + "NameEnvVar": "FELIX_PolicySyncPathPrefix", + "NameYAML": "policySyncPathPrefix", + "NameGoAPI": "PolicySyncPathPrefix", + "StringSchema": "Path to file", + "StringSchemaHTML": "Path to file", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to by Felix to communicate policy changes to external services,\nlike Application layer policy.", + "DescriptionHTML": "

Used to by Felix to communicate policy changes to external services,\nlike Application layer policy.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "ProgramClusterRoutes", + "NameEnvVar": "FELIX_ProgramClusterRoutes", + "NameYAML": "programClusterRoutes", + "NameGoAPI": "ProgramClusterRoutes", + "StringSchema": "One of: `Disabled`, `Enabled`", + "StringSchemaHTML": "One of: Disabled, Enabled", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Disabled\"`, `\"Enabled\"`.", + "YAMLEnumValues": [ + "Disabled", + "Enabled" + ], + "YAMLSchemaHTML": "One of: \"Disabled\", \"Enabled\".", + "YAMLDefault": "Disabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls how a cluster node gets a route to a workload on another node,\nwhen that workload's IP comes from an IP Pool with vxlanMode: Never. When ProgramClusterRoutes is Disabled,\nit is expected that confd and BIRD will program that route. When ProgramClusterRoutes is Enabled, Felix program that route.\nFelix always programs such routes for IP Pools with vxlanMode: Always or vxlanMode: CrossSubnet.", + "DescriptionHTML": "

Controls how a cluster node gets a route to a workload on another node,\nwhen that workload's IP comes from an IP Pool with vxlanMode: Never. When ProgramClusterRoutes is Disabled,\nit is expected that confd and BIRD will program that route. When ProgramClusterRoutes is Enabled, Felix program that route.\nFelix always programs such routes for IP Pools with vxlanMode: Always or vxlanMode: CrossSubnet.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "RemoveExternalRoutes", + "NameEnvVar": "FELIX_RemoveExternalRoutes", + "NameYAML": "removeExternalRoutes", + "NameGoAPI": "RemoveExternalRoutes", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix will remove unexpected routes to workload interfaces. Felix will\nalways clean up expected routes that use the configured DeviceRouteProtocol. To add your own routes, you must\nuse a distinct protocol (in addition to setting this field to false).", + "DescriptionHTML": "

Controls whether Felix will remove unexpected routes to workload interfaces. Felix will\nalways clean up expected routes that use the configured DeviceRouteProtocol. To add your own routes, you must\nuse a distinct protocol (in addition to setting this field to false).

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "RequireMTUFile", + "NameEnvVar": "FELIX_RequireMTUFile", + "NameYAML": "requireMTUFile", + "NameGoAPI": "RequireMTUFile", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies whether mtu file is required to start the felix.\nOptional as to keep the same as previous behavior.", + "DescriptionHTML": "

Specifies whether mtu file is required to start the felix.\nOptional as to keep the same as previous behavior.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "RouteRefreshInterval", + "NameEnvVar": "FELIX_RouteRefreshInterval", + "NameYAML": "routeRefreshInterval", + "NameGoAPI": "RouteRefreshInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "90", + "ParsedDefault": "1m30s", + "ParsedDefaultJSON": "90000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1m30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The period at which Felix re-checks the routes\nin the dataplane to ensure that no other process has accidentally broken Calico's rules.\nSet to 0 to disable route refresh.", + "DescriptionHTML": "

The period at which Felix re-checks the routes\nin the dataplane to ensure that no other process has accidentally broken Calico's rules.\nSet to 0 to disable route refresh.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "RouteSource", + "NameEnvVar": "FELIX_RouteSource", + "NameYAML": "routeSource", + "NameGoAPI": "RouteSource", + "StringSchema": "One of: `CalicoIPAM`, `WorkloadIPs`", + "StringSchemaHTML": "One of: CalicoIPAM, WorkloadIPs", + "StringDefault": "CalicoIPAM", + "ParsedDefault": "CalicoIPAM", + "ParsedDefaultJSON": "\"CalicoIPAM\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `CalicoIPAM`, `WorkloadIPs`.", + "YAMLEnumValues": [ + "CalicoIPAM", + "WorkloadIPs" + ], + "YAMLSchemaHTML": "One of: CalicoIPAM, WorkloadIPs.", + "YAMLDefault": "CalicoIPAM", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures where Felix gets its routing information.\n- WorkloadIPs: use workload endpoints to construct routes.\n- CalicoIPAM: the default - use IPAM data to construct routes.", + "DescriptionHTML": "

Configures where Felix gets its routing information.\n- WorkloadIPs: use workload endpoints to construct routes.\n- CalicoIPAM: the default - use IPAM data to construct routes.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "RouteSyncDisabled", + "NameEnvVar": "FELIX_RouteSyncDisabled", + "NameYAML": "routeSyncDisabled", + "NameGoAPI": "RouteSyncDisabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Will disable all operations performed on the route table. Set to true to\nrun in network-policy mode only.", + "DescriptionHTML": "

Will disable all operations performed on the route table. Set to true to\nrun in network-policy mode only.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "RouteTableRange", + "NameEnvVar": "FELIX_RouteTableRange", + "NameYAML": "routeTableRange", + "NameGoAPI": "RouteTableRange", + "StringSchema": "Range of route table indices `n-m`, where `n` and `m` are integers in [0,250].", + "StringSchemaHTML": "Range of route table indices n-m, where n and m are integers in [0,250].", + "StringDefault": "", + "ParsedDefault": "{0 0}", + "ParsedDefaultJSON": "{\"Min\":0,\"Max\":0}", + "ParsedType": "idalloc.IndexRange", + "YAMLType": "object", + "YAMLSchema": "Route table range: `{min:, max}`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Route table range: {min:<n>, max<m>}.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Deprecated in favor of RouteTableRanges.\nCalico programs additional Linux route tables for various purposes.\nRouteTableRange specifies the indices of the route tables that Calico should use.", + "DescriptionHTML": "

Deprecated in favor of RouteTableRanges.\nCalico programs additional Linux route tables for various purposes.\nRouteTableRange specifies the indices of the route tables that Calico should use.

", + "UserEditable": true, + "GoType": "*v3.RouteTableRange", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "RouteTableRanges", + "NameEnvVar": "FELIX_RouteTableRanges", + "NameYAML": "routeTableRanges", + "NameGoAPI": "RouteTableRanges", + "StringSchema": "Comma or space-delimited list of route table ranges of the form `n-m` where `n` and `m` are integers in [0,4294967295]. The sum of the sizes of all ranges may not exceed 65535.", + "StringSchemaHTML": "Comma or space-delimited list of route table ranges of the form n-m where n and m are integers in [0,4294967295]. The sum of the sizes of all ranges may not exceed 65535.", + "StringDefault": "", + "ParsedDefault": "[]", + "ParsedDefaultJSON": "null", + "ParsedType": "[]idalloc.IndexRange", + "YAMLType": "array", + "YAMLSchema": "List of route table ranges: `[{min:, max}, ...]`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of route table ranges: [{min:<n>, max<m>}, ...].", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Calico programs additional Linux route tables for various purposes.\nRouteTableRanges specifies a set of table index ranges that Calico should use.\nDeprecates`RouteTableRange`, overrides `RouteTableRange`.", + "DescriptionHTML": "

Calico programs additional Linux route tables for various purposes.\nRouteTableRanges specifies a set of table index ranges that Calico should use.\nDeprecatesRouteTableRange, overrides RouteTableRange.

", + "UserEditable": true, + "GoType": "*v3.RouteTableRanges", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "ServiceLoopPrevention", + "NameEnvVar": "FELIX_ServiceLoopPrevention", + "NameYAML": "serviceLoopPrevention", + "NameGoAPI": "ServiceLoopPrevention", + "StringSchema": "One of: `Disabled`, `Drop`, `Reject`", + "StringSchemaHTML": "One of: Disabled, Drop, Reject", + "StringDefault": "Drop", + "ParsedDefault": "Drop", + "ParsedDefaultJSON": "\"Drop\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `Drop`, `Reject`.", + "YAMLEnumValues": [ + "Disabled", + "Drop", + "Reject" + ], + "YAMLSchemaHTML": "One of: Disabled, Drop, Reject.", + "YAMLDefault": "Drop", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "When service IP advertisement is enabled, prevent routing loops to service IPs that are\nnot in use, by dropping or rejecting packets that do not get DNAT'd by kube-proxy.\nUnless set to \"Disabled\", in which case such routing loops continue to be allowed.", + "DescriptionHTML": "

When service IP advertisement is enabled, prevent routing loops to service IPs that are\nnot in use, by dropping or rejecting packets that do not get DNAT'd by kube-proxy.\nUnless set to \"Disabled\", in which case such routing loops continue to be allowed.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "SidecarAccelerationEnabled", + "NameEnvVar": "FELIX_SidecarAccelerationEnabled", + "NameYAML": "sidecarAccelerationEnabled", + "NameGoAPI": "SidecarAccelerationEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Enables experimental sidecar acceleration.", + "DescriptionHTML": "

Enables experimental sidecar acceleration.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "UseInternalDataplaneDriver", + "NameEnvVar": "FELIX_UseInternalDataplaneDriver", + "NameYAML": "useInternalDataplaneDriver", + "NameGoAPI": "UseInternalDataplaneDriver", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "If true, Felix will use its internal dataplane programming logic. If false, it\nwill launch an external dataplane driver and communicate with it over protobuf.", + "DescriptionHTML": "

If true, Felix will use its internal dataplane programming logic. If false, it\nwill launch an external dataplane driver and communicate with it over protobuf.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "WAFEventLogsFileDirectory", + "NameEnvVar": "FELIX_WAFEventLogsFileDirectory", + "NameYAML": "wafEventLogsFileDirectory", + "NameGoAPI": "WAFEventLogsFileDirectory", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "/var/log/calico/waf", + "ParsedDefault": "/var/log/calico/waf", + "ParsedDefaultJSON": "\"/var/log/calico/waf\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "/var/log/calico/waf", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the directory where WAFEvent log files are stored.", + "DescriptionHTML": "

Sets the directory where WAFEvent log files are stored.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "WAFEventLogsFileEnabled", + "NameEnvVar": "FELIX_WAFEventLogsFileEnabled", + "NameYAML": "wafEventLogsFileEnabled", + "NameGoAPI": "WAFEventLogsFileEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls logging WAFEvent logs to a file. If false no WAFEvent logging to file will occur.", + "DescriptionHTML": "

Controls logging WAFEvent logs to a file. If false no WAFEvent logging to file will occur.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "WAFEventLogsFileMaxFileSizeMB", + "NameEnvVar": "FELIX_WAFEventLogsFileMaxFileSizeMB", + "NameYAML": "wafEventLogsFileMaxFileSizeMB", + "NameGoAPI": "WAFEventLogsFileMaxFileSizeMB", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "100", + "ParsedDefault": "100", + "ParsedDefaultJSON": "100", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "100", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the max size in MB of WAFEvent log files before rotation.", + "DescriptionHTML": "

Sets the max size in MB of WAFEvent log files before rotation.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "WAFEventLogsFileMaxFiles", + "NameEnvVar": "FELIX_WAFEventLogsFileMaxFiles", + "NameYAML": "wafEventLogsFileMaxFiles", + "NameGoAPI": "WAFEventLogsFileMaxFiles", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "5", + "ParsedDefault": "5", + "ParsedDefaultJSON": "5", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "5", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the number of WAFEvent log files to keep.", + "DescriptionHTML": "

Sets the number of WAFEvent log files to keep.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "WAFEventLogsFlushInterval", + "NameEnvVar": "FELIX_WAFEventLogsFlushInterval", + "NameYAML": "wafEventLogsFlushInterval", + "NameGoAPI": "WAFEventLogsFlushInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "15", + "ParsedDefault": "15s", + "ParsedDefaultJSON": "15000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "15s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures the interval at which Felix exports WAFEvent logs.", + "DescriptionHTML": "

Configures the interval at which Felix exports WAFEvent logs.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Common", + "GroupWithSortPrefix": "10 Dataplane: Common", + "NameConfigFile": "WorkloadSourceSpoofing", + "NameEnvVar": "FELIX_WorkloadSourceSpoofing", + "NameYAML": "workloadSourceSpoofing", + "NameGoAPI": "WorkloadSourceSpoofing", + "StringSchema": "One of: `Any`, `Disabled`", + "StringSchemaHTML": "One of: Any, Disabled", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Any`, `Disabled`.", + "YAMLEnumValues": [ + "Any", + "Disabled" + ], + "YAMLSchemaHTML": "One of: Any, Disabled.", + "YAMLDefault": "Disabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source IP\naddress that is not theirs. This is disabled by default. When set to \"Any\", pods can request any prefix.", + "DescriptionHTML": "

Controls whether pods can use the allowedSourcePrefixes annotation to send traffic with a source IP\naddress that is not theirs. This is disabled by default. When set to \"Any\", pods can request any prefix.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Dataplane: iptables", + "Fields": [ + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IpsetsRefreshInterval", + "NameEnvVar": "FELIX_IpsetsRefreshInterval", + "NameYAML": "ipsetsRefreshInterval", + "NameGoAPI": "IpsetsRefreshInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "90", + "ParsedDefault": "1m30s", + "ParsedDefaultJSON": "90000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1m30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the period at which Felix re-checks all IP sets to look for discrepancies.\nSet to 0 to disable the periodic refresh.", + "DescriptionHTML": "

Controls the period at which Felix re-checks all IP sets to look for discrepancies.\nSet to 0 to disable the periodic refresh.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesBackend", + "NameEnvVar": "FELIX_IptablesBackend", + "NameYAML": "iptablesBackend", + "NameGoAPI": "IptablesBackend", + "StringSchema": "One of: `auto`, `legacy`, `nft`", + "StringSchemaHTML": "One of: auto, legacy, nft", + "StringDefault": "auto", + "ParsedDefault": "auto", + "ParsedDefaultJSON": "\"auto\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Auto\"`, `\"Legacy\"`, `\"NFT\"`.", + "YAMLEnumValues": [ + "Auto", + "Auto", + "Legacy", + "Legacy", + "NFT", + "NFT" + ], + "YAMLSchemaHTML": "One of: \"Auto\", \"Legacy\", \"NFT\".", + "YAMLDefault": "Auto", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls which backend of iptables will be used. The default is `Auto`.\n\nWarning: changing this on a running system can leave \"orphaned\" rules in the \"other\" backend. These\nshould be cleaned up to avoid confusing interactions.", + "DescriptionHTML": "

Controls which backend of iptables will be used. The default is Auto.

\n

Warning: changing this on a running system can leave \"orphaned\" rules in the \"other\" backend. These\nshould be cleaned up to avoid confusing interactions.

", + "UserEditable": true, + "GoType": "*v3.IptablesBackend", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesFilterAllowAction", + "NameEnvVar": "FELIX_IptablesFilterAllowAction", + "NameYAML": "iptablesFilterAllowAction", + "NameGoAPI": "IptablesFilterAllowAction", + "StringSchema": "One of: `ACCEPT`, `RETURN`", + "StringSchemaHTML": "One of: ACCEPT, RETURN", + "StringDefault": "ACCEPT", + "ParsedDefault": "ACCEPT", + "ParsedDefaultJSON": "\"ACCEPT\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Accept`, `Return`.", + "YAMLEnumValues": [ + "Accept", + "Return" + ], + "YAMLSchemaHTML": "One of: Accept, Return.", + "YAMLDefault": "Accept", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls what happens to traffic that is accepted by a Felix policy chain in the\niptables filter table (which is used for \"normal\" policy). The default will immediately `Accept` the traffic. Use\n`Return` to send the traffic back up to the system chains for further processing.", + "DescriptionHTML": "

Controls what happens to traffic that is accepted by a Felix policy chain in the\niptables filter table (which is used for \"normal\" policy). The default will immediately Accept the traffic. Use\nReturn to send the traffic back up to the system chains for further processing.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesFilterDenyAction", + "NameEnvVar": "FELIX_IptablesFilterDenyAction", + "NameYAML": "iptablesFilterDenyAction", + "NameGoAPI": "IptablesFilterDenyAction", + "StringSchema": "One of: `DROP`, `REJECT`", + "StringSchemaHTML": "One of: DROP, REJECT", + "StringDefault": "DROP", + "ParsedDefault": "DROP", + "ParsedDefaultJSON": "\"DROP\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Drop`, `Reject`.", + "YAMLEnumValues": [ + "Drop", + "Reject" + ], + "YAMLSchemaHTML": "One of: Drop, Reject.", + "YAMLDefault": "Drop", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls what happens to traffic that is denied by network policy. By default Calico blocks traffic\nwith an iptables \"DROP\" action. If you want to use \"REJECT\" action instead you can configure it in here.", + "DescriptionHTML": "

Controls what happens to traffic that is denied by network policy. By default Calico blocks traffic\nwith an iptables \"DROP\" action. If you want to use \"REJECT\" action instead you can configure it in here.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesLockProbeIntervalMillis", + "NameEnvVar": "FELIX_IptablesLockProbeIntervalMillis", + "NameYAML": "iptablesLockProbeInterval", + "NameGoAPI": "IptablesLockProbeInterval", + "StringSchema": "Milliseconds (floating point)", + "StringSchemaHTML": "Milliseconds (floating point)", + "StringDefault": "50", + "ParsedDefault": "50ms", + "ParsedDefaultJSON": "50000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "50ms", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures the interval between attempts to claim\nthe xtables lock. Shorter intervals are more responsive but use more CPU.", + "DescriptionHTML": "

Configures the interval between attempts to claim\nthe xtables lock. Shorter intervals are more responsive but use more CPU.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesMangleAllowAction", + "NameEnvVar": "FELIX_IptablesMangleAllowAction", + "NameYAML": "iptablesMangleAllowAction", + "NameGoAPI": "IptablesMangleAllowAction", + "StringSchema": "One of: `ACCEPT`, `RETURN`", + "StringSchemaHTML": "One of: ACCEPT, RETURN", + "StringDefault": "ACCEPT", + "ParsedDefault": "ACCEPT", + "ParsedDefaultJSON": "\"ACCEPT\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Accept`, `Return`.", + "YAMLEnumValues": [ + "Accept", + "Return" + ], + "YAMLSchemaHTML": "One of: Accept, Return.", + "YAMLDefault": "Accept", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls what happens to traffic that is accepted by a Felix policy chain in the\niptables mangle table (which is used for \"pre-DNAT\" policy). The default will immediately `Accept` the traffic.\nUse `Return` to send the traffic back up to the system chains for further processing.", + "DescriptionHTML": "

Controls what happens to traffic that is accepted by a Felix policy chain in the\niptables mangle table (which is used for \"pre-DNAT\" policy). The default will immediately Accept the traffic.\nUse Return to send the traffic back up to the system chains for further processing.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesMarkMask", + "NameEnvVar": "FELIX_IptablesMarkMask", + "NameYAML": "iptablesMarkMask", + "NameGoAPI": "IptablesMarkMask", + "StringSchema": "32-bit bitmask (hex or deccimal allowed) with at least 2 bits set, example: `0xffff0000`", + "StringSchemaHTML": "32-bit bitmask (hex or deccimal allowed) with at least 2 bits set, example: 0xffff0000", + "StringDefault": "0xffff0000", + "ParsedDefault": "4294901760", + "ParsedDefaultJSON": "4294901760", + "ParsedType": "uint32", + "YAMLType": "integer", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "0xffff0000", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "The mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal\nnumber with at least 8 bits set, none of which clash with any other mark bits in use on the system.", + "DescriptionHTML": "

The mask that Felix selects its IPTables Mark bits from. Should be a 32 bit hexadecimal\nnumber with at least 8 bits set, none of which clash with any other mark bits in use on the system.

", + "UserEditable": true, + "GoType": "*int64", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesNATOutgoingInterfaceFilter", + "NameEnvVar": "FELIX_IptablesNATOutgoingInterfaceFilter", + "NameYAML": "iptablesNATOutgoingInterfaceFilter", + "NameGoAPI": "IptablesNATOutgoingInterfaceFilter", + "StringSchema": "String matching regex `^[a-zA-Z0-9:._+-]{1,15}$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9:._+-]{1,15}$", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "This parameter can be used to limit the host interfaces on which Calico will apply SNAT to traffic leaving a\nCalico IPAM pool with \"NAT outgoing\" enabled. This can be useful if you have a main data interface, where\ntraffic should be SNATted and a secondary device (such as the docker bridge) which is local to the host and\ndoesn't require SNAT. This parameter uses the iptables interface matching syntax, which allows + as a\nwildcard. Most users will not need to set this. Example: if your data interfaces are eth0 and eth1 and you\nwant to exclude the docker bridge, you could set this to eth+.", + "DescriptionHTML": "

This parameter can be used to limit the host interfaces on which Calico will apply SNAT to traffic leaving a\nCalico IPAM pool with \"NAT outgoing\" enabled. This can be useful if you have a main data interface, where\ntraffic should be SNATted and a secondary device (such as the docker bridge) which is local to the host and\ndoesn't require SNAT. This parameter uses the iptables interface matching syntax, which allows + as a\nwildcard. Most users will not need to set this. Example: if your data interfaces are eth0 and eth1 and you\nwant to exclude the docker bridge, you could set this to eth+.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesPostWriteCheckIntervalSecs", + "NameEnvVar": "FELIX_IptablesPostWriteCheckIntervalSecs", + "NameYAML": "iptablesPostWriteCheckInterval", + "NameGoAPI": "IptablesPostWriteCheckInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "5", + "ParsedDefault": "5s", + "ParsedDefaultJSON": "5000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "5s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The period after Felix has done a write\nto the dataplane that it schedules an extra read back in order to check the write was not\nclobbered by another process. This should only occur if another application on the system\ndoesn't respect the iptables lock.", + "DescriptionHTML": "

The period after Felix has done a write\nto the dataplane that it schedules an extra read back in order to check the write was not\nclobbered by another process. This should only occur if another application on the system\ndoesn't respect the iptables lock.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "IptablesRefreshInterval", + "NameEnvVar": "FELIX_IptablesRefreshInterval", + "NameYAML": "iptablesRefreshInterval", + "NameGoAPI": "IptablesRefreshInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "180", + "ParsedDefault": "3m0s", + "ParsedDefaultJSON": "180000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "3m0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The period at which Felix re-checks the IP sets\nin the dataplane to ensure that no other process has accidentally broken Calico's rules.\nSet to 0 to disable IP sets refresh. Note: the default for this value is lower than the\nother refresh intervals as a workaround for a Linux kernel bug that was fixed in kernel\nversion 4.11. If you are using v4.11 or greater you may want to set this to, a higher value\nto reduce Felix CPU usage.", + "DescriptionHTML": "

The period at which Felix re-checks the IP sets\nin the dataplane to ensure that no other process has accidentally broken Calico's rules.\nSet to 0 to disable IP sets refresh. Note: the default for this value is lower than the\nother refresh intervals as a workaround for a Linux kernel bug that was fixed in kernel\nversion 4.11. If you are using v4.11 or greater you may want to set this to, a higher value\nto reduce Felix CPU usage.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "KubeNodePortRanges", + "NameEnvVar": "FELIX_KubeNodePortRanges", + "NameYAML": "kubeNodePortRanges", + "NameGoAPI": "KubeNodePortRanges", + "StringSchema": "List of port ranges: comma-delimited list of either single numbers in range [0,65535] or a ranges of numbers `n:m`", + "StringSchemaHTML": "List of port ranges: comma-delimited list of either single numbers in range [0,65535] or a ranges of numbers n:m", + "StringDefault": "30000:32767", + "ParsedDefault": "[30000:32767]", + "ParsedDefaultJSON": "[\"30000:32767\"]", + "ParsedType": "[]numorstring.Port", + "YAMLType": "array", + "YAMLSchema": "List of ports: `[, ...]` where `` is a port number (integer) or range (string), for example `80`, `8080:8089`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of ports: [<port>, ...] where <port> is a port number (integer) or range (string), for example 80, 8080:8089.", + "YAMLDefault": "[\"30000:32767\"]", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Holds list of port ranges used for service node ports. Only used if felix detects kube-proxy running in ipvs mode.\nFelix uses these ranges to separate host and workload traffic. .", + "DescriptionHTML": "

Holds list of port ranges used for service node ports. Only used if felix detects kube-proxy running in ipvs mode.\nFelix uses these ranges to separate host and workload traffic. .

", + "UserEditable": true, + "GoType": "*[]numorstring.Port", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: iptables", + "GroupWithSortPrefix": "20 Dataplane: iptables", + "NameConfigFile": "MaxIpsetSize", + "NameEnvVar": "FELIX_MaxIpsetSize", + "NameYAML": "maxIpsetSize", + "NameGoAPI": "MaxIpsetSize", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "1048576", + "ParsedDefault": "1048576", + "ParsedDefaultJSON": "1048576", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "1048576", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The maximum number of IP addresses that can be stored in an IP set. Not applicable\nif using the nftables backend.", + "DescriptionHTML": "

The maximum number of IP addresses that can be stored in an IP set. Not applicable\nif using the nftables backend.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Dataplane: nftables", + "Fields": [ + { + "Group": "Dataplane: nftables", + "GroupWithSortPrefix": "21 Dataplane: nftables", + "NameConfigFile": "NftablesFilterAllowAction", + "NameEnvVar": "FELIX_NftablesFilterAllowAction", + "NameYAML": "nftablesFilterAllowAction", + "NameGoAPI": "NftablesFilterAllowAction", + "StringSchema": "One of: `ACCEPT`, `RETURN`", + "StringSchemaHTML": "One of: ACCEPT, RETURN", + "StringDefault": "ACCEPT", + "ParsedDefault": "ACCEPT", + "ParsedDefaultJSON": "\"ACCEPT\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Accept`, `Return`.", + "YAMLEnumValues": [ + "Accept", + "Return" + ], + "YAMLSchemaHTML": "One of: Accept, Return.", + "YAMLDefault": "Accept", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls the nftables action that Felix uses to represent the \"allow\" policy verdict\nin the filter table. The default is to `ACCEPT` the traffic, which is a terminal action. Alternatively,\n`RETURN` can be used to return the traffic back to the top-level chain for further processing by your rules.", + "DescriptionHTML": "

Controls the nftables action that Felix uses to represent the \"allow\" policy verdict\nin the filter table. The default is to ACCEPT the traffic, which is a terminal action. Alternatively,\nRETURN can be used to return the traffic back to the top-level chain for further processing by your rules.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: nftables", + "GroupWithSortPrefix": "21 Dataplane: nftables", + "NameConfigFile": "NftablesFilterDenyAction", + "NameEnvVar": "FELIX_NftablesFilterDenyAction", + "NameYAML": "nftablesFilterDenyAction", + "NameGoAPI": "NftablesFilterDenyAction", + "StringSchema": "One of: `DROP`, `REJECT`", + "StringSchemaHTML": "One of: DROP, REJECT", + "StringDefault": "DROP", + "ParsedDefault": "DROP", + "ParsedDefaultJSON": "\"DROP\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Drop`, `Reject`.", + "YAMLEnumValues": [ + "Drop", + "Reject" + ], + "YAMLSchemaHTML": "One of: Drop, Reject.", + "YAMLDefault": "Drop", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls what happens to traffic that is denied by network policy. By default, Calico\nblocks traffic with a \"drop\" action. If you want to use a \"reject\" action instead you can configure it here.", + "DescriptionHTML": "

Controls what happens to traffic that is denied by network policy. By default, Calico\nblocks traffic with a \"drop\" action. If you want to use a \"reject\" action instead you can configure it here.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: nftables", + "GroupWithSortPrefix": "21 Dataplane: nftables", + "NameConfigFile": "NftablesMangleAllowAction", + "NameEnvVar": "FELIX_NftablesMangleAllowAction", + "NameYAML": "nftablesMangleAllowAction", + "NameGoAPI": "NftablesMangleAllowAction", + "StringSchema": "One of: `ACCEPT`, `RETURN`", + "StringSchemaHTML": "One of: ACCEPT, RETURN", + "StringDefault": "ACCEPT", + "ParsedDefault": "ACCEPT", + "ParsedDefaultJSON": "\"ACCEPT\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Accept`, `Return`.", + "YAMLEnumValues": [ + "Accept", + "Return" + ], + "YAMLSchemaHTML": "One of: Accept, Return.", + "YAMLDefault": "Accept", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "Controls the nftables action that Felix uses to represent the \"allow\" policy verdict\nin the mangle table. The default is to `ACCEPT` the traffic, which is a terminal action. Alternatively,\n`RETURN` can be used to return the traffic back to the top-level chain for further processing by your rules.", + "DescriptionHTML": "

Controls the nftables action that Felix uses to represent the \"allow\" policy verdict\nin the mangle table. The default is to ACCEPT the traffic, which is a terminal action. Alternatively,\nRETURN can be used to return the traffic back to the top-level chain for further processing by your rules.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: nftables", + "GroupWithSortPrefix": "21 Dataplane: nftables", + "NameConfigFile": "NftablesMarkMask", + "NameEnvVar": "FELIX_NftablesMarkMask", + "NameYAML": "nftablesMarkMask", + "NameGoAPI": "NftablesMarkMask", + "StringSchema": "32-bit bitmask (hex or deccimal allowed) with at least 2 bits set, example: `0xffff0000`", + "StringSchemaHTML": "32-bit bitmask (hex or deccimal allowed) with at least 2 bits set, example: 0xffff0000", + "StringDefault": "0xffff0000", + "ParsedDefault": "4294901760", + "ParsedDefaultJSON": "4294901760", + "ParsedType": "uint32", + "YAMLType": "integer", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "0xffff0000", + "Required": true, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "The mask that Felix selects its nftables Mark bits from. Should be a 32 bit hexadecimal\nnumber with at least 8 bits set, none of which clash with any other mark bits in use on the system.", + "DescriptionHTML": "

The mask that Felix selects its nftables Mark bits from. Should be a 32 bit hexadecimal\nnumber with at least 8 bits set, none of which clash with any other mark bits in use on the system.

", + "UserEditable": true, + "GoType": "*int64", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: nftables", + "GroupWithSortPrefix": "21 Dataplane: nftables", + "NameConfigFile": "NftablesRefreshInterval", + "NameEnvVar": "FELIX_NftablesRefreshInterval", + "NameYAML": "nftablesRefreshInterval", + "NameGoAPI": "NftablesRefreshInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "180", + "ParsedDefault": "3m0s", + "ParsedDefaultJSON": "180000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "3m0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the interval at which Felix periodically refreshes the nftables rules.", + "DescriptionHTML": "

Controls the interval at which Felix periodically refreshes the nftables rules.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Dataplane: eBPF", + "Fields": [ + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFAttachType", + "NameEnvVar": "FELIX_BPFAttachType", + "NameYAML": "bpfAttachType", + "NameGoAPI": "BPFAttachType", + "StringSchema": "One of: `TCX`, `TC`", + "StringSchemaHTML": "One of: TCX, TC", + "StringDefault": "TCX", + "ParsedDefault": "TCX", + "ParsedDefaultJSON": "\"TCX\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"TC\"`, `\"TCX\"`.", + "YAMLEnumValues": [ + "TC", + "TCX" + ], + "YAMLSchemaHTML": "One of: \"TC\", \"TCX\".", + "YAMLDefault": "TCX", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls how are the BPF programs at the network interfaces attached.\nBy default `TCX` is used where available to enable easier coexistence with 3rd party programs.\n`TC` can force the legacy method of attaching via a qdisc. `TCX` falls back to `TC` if `TCX` is not available.", + "DescriptionHTML": "

Controls how are the BPF programs at the network interfaces attached.\nBy default TCX is used where available to enable easier coexistence with 3rd party programs.\nTC can force the legacy method of attaching via a qdisc. TCX falls back to TC if TCX is not available.

", + "UserEditable": true, + "GoType": "*v3.BPFAttachOption", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFCTLBLogFilter", + "NameEnvVar": "FELIX_BPFCTLBLogFilter", + "NameYAML": "bpfCTLBLogFilter", + "NameGoAPI": "BPFCTLBLogFilter", + "StringSchema": "One of: `all`", + "StringSchemaHTML": "One of: all", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies, what is logged by connect time load balancer when BPFLogLevel is\ndebug. Currently has to be specified as 'all' when BPFLogFilters is set\nto see CTLB logs.", + "DescriptionHTML": "

Specifies, what is logged by connect time load balancer when BPFLogLevel is\ndebug. Currently has to be specified as 'all' when BPFLogFilters is set\nto see CTLB logs.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFConnectTimeLoadBalancing", + "NameEnvVar": "FELIX_BPFConnectTimeLoadBalancing", + "NameYAML": "bpfConnectTimeLoadBalancing", + "NameGoAPI": "BPFConnectTimeLoadBalancing", + "StringSchema": "One of: `Disabled`, `Enabled`, `TCP`", + "StringSchemaHTML": "One of: Disabled, Enabled, TCP", + "StringDefault": "TCP", + "ParsedDefault": "TCP", + "ParsedDefaultJSON": "\"TCP\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Disabled\"`, `\"Enabled\"`, `\"TCP\"`.", + "YAMLEnumValues": [ + "Disabled", + "Enabled", + "TCP" + ], + "YAMLSchemaHTML": "One of: \"Disabled\", \"Enabled\", \"TCP\".", + "YAMLDefault": "TCP", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "When in BPF mode, controls whether Felix installs the connect-time load\nbalancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services\nand it improves the performance of pod-to-service connections.When set to TCP, connect time load balancing\nis available only for services with TCP ports.", + "DescriptionHTML": "

When in BPF mode, controls whether Felix installs the connect-time load\nbalancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services\nand it improves the performance of pod-to-service connections.When set to TCP, connect time load balancing\nis available only for services with TCP ports.

", + "UserEditable": true, + "GoType": "*v3.BPFConnectTimeLBType", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFConnectTimeLoadBalancingEnabled", + "NameEnvVar": "FELIX_BPFConnectTimeLoadBalancingEnabled", + "NameYAML": "bpfConnectTimeLoadBalancingEnabled", + "NameGoAPI": "BPFConnectTimeLoadBalancingEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "When in BPF mode, controls whether Felix installs the connection-time load\nbalancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services\nand it improves the performance of pod-to-service connections. The only reason to disable it is for debugging\npurposes.\n\nDeprecated: Use BPFConnectTimeLoadBalancing.", + "DescriptionHTML": "

When in BPF mode, controls whether Felix installs the connection-time load\nbalancer. The connect-time load balancer is required for the host to be able to reach Kubernetes services\nand it improves the performance of pod-to-service connections. The only reason to disable it is for debugging\npurposes.

\n

Deprecated: Use BPFConnectTimeLoadBalancing.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFConntrackCleanupMode", + "NameEnvVar": "FELIX_BPFConntrackCleanupMode", + "NameYAML": "bpfConntrackMode", + "NameGoAPI": "BPFConntrackCleanupMode", + "StringSchema": "One of: `Auto`, `BPFProgram`, `Userspace`", + "StringSchemaHTML": "One of: Auto, BPFProgram, Userspace", + "StringDefault": "Auto", + "ParsedDefault": "Auto", + "ParsedDefaultJSON": "\"Auto\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Auto\"`, `\"BPFProgram\"`, `\"Userspace\"`.", + "YAMLEnumValues": [ + "Auto", + "BPFProgram", + "Userspace" + ], + "YAMLSchemaHTML": "One of: \"Auto\", \"BPFProgram\", \"Userspace\".", + "YAMLDefault": "Auto", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls how BPF conntrack entries are cleaned up. `Auto` will use a BPF program if supported,\nfalling back to userspace if not. `Userspace` will always use the userspace cleanup code. `BPFProgram` will\nalways use the BPF program (failing if not supported).\n\n/To be deprecated in future versions as conntrack map type changed to\nlru_hash and userspace cleanup is the only mode that is supported.", + "DescriptionHTML": "

Controls how BPF conntrack entries are cleaned up. Auto will use a BPF program if supported,\nfalling back to userspace if not. Userspace will always use the userspace cleanup code. BPFProgram will\nalways use the BPF program (failing if not supported).

\n

/To be deprecated in future versions as conntrack map type changed to\nlru_hash and userspace cleanup is the only mode that is supported.

", + "UserEditable": true, + "GoType": "*v3.BPFConntrackMode", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFConntrackLogLevel", + "NameEnvVar": "FELIX_BPFConntrackLogLevel", + "NameYAML": "bpfConntrackLogLevel", + "NameGoAPI": "BPFConntrackLogLevel", + "StringSchema": "One of: `debug`, `off`", + "StringSchemaHTML": "One of: debug, off", + "StringDefault": "off", + "ParsedDefault": "off", + "ParsedDefaultJSON": "\"off\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Debug\"`, `\"Off\"`.", + "YAMLEnumValues": [ + "Debug", + "Off" + ], + "YAMLSchemaHTML": "One of: \"Debug\", \"Off\".", + "YAMLDefault": "Off", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the log level of the BPF conntrack cleanup program, which runs periodically\nto clean up expired BPF conntrack entries.\n.", + "DescriptionHTML": "

Controls the log level of the BPF conntrack cleanup program, which runs periodically\nto clean up expired BPF conntrack entries.\n.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFConntrackTimeouts", + "NameEnvVar": "FELIX_BPFConntrackTimeouts", + "NameYAML": "bpfConntrackTimeouts", + "NameGoAPI": "BPFConntrackTimeouts", + "StringSchema": "Comma-delimited list of key=value pairs", + "StringSchemaHTML": "Comma-delimited list of key=value pairs", + "StringDefault": "CreationGracePeriod=10s,TCPSynSent=20s,TCPEstablished=1h,TCPFinsSeen=Auto,TCPResetSeen=40s,UDPTimeout=60s,GenericTimeout=10m,ICMPTimeout=5s", + "ParsedDefault": "map[CreationGracePeriod:10s GenericTimeout:10m ICMPTimeout:5s TCPEstablished:1h TCPFinsSeen:Auto TCPResetSeen:40s TCPSynSent:20s UDPTimeout:60s]", + "ParsedDefaultJSON": "{\"CreationGracePeriod\":\"10s\",\"GenericTimeout\":\"10m\",\"ICMPTimeout\":\"5s\",\"TCPEstablished\":\"1h\",\"TCPFinsSeen\":\"Auto\",\"TCPResetSeen\":\"40s\",\"TCPSynSent\":\"20s\",\"UDPTimeout\":\"60s\"}", + "ParsedType": "map[string]string", + "YAMLType": "object", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "BPFConntrackTimers overrides the default values for the specified conntrack timer if\nset. Each value can be either a duration or `Auto` to pick the value from\na Linux conntrack timeout.\n\nConfigurable timers are: CreationGracePeriod, TCPSynSent,\nTCPEstablished, TCPFinsSeen, TCPResetSeen, UDPTimeout, GenericTimeout,\nICMPTimeout.\n\nUnset values are replaced by the default values with a warning log for\nincorrect values.", + "DescriptionHTML": "

BPFConntrackTimers overrides the default values for the specified conntrack timer if\nset. Each value can be either a duration or Auto to pick the value from\na Linux conntrack timeout.

\n

Configurable timers are: CreationGracePeriod, TCPSynSent,\nTCPEstablished, TCPFinsSeen, TCPResetSeen, UDPTimeout, GenericTimeout,\nICMPTimeout.

\n

Unset values are replaced by the default values with a warning log for\nincorrect values.

", + "UserEditable": true, + "GoType": "*v3.BPFConntrackTimeouts", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFDNSPolicyMode", + "NameEnvVar": "FELIX_BPFDNSPolicyMode", + "NameYAML": "bpfDNSPolicyMode", + "NameGoAPI": "BPFDNSPolicyMode", + "StringSchema": "One of: `Inline`, `NoDelay`", + "StringSchemaHTML": "One of: Inline, NoDelay", + "StringDefault": "Inline", + "ParsedDefault": "Inline", + "ParsedDefaultJSON": "\"Inline\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "Inline", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies how DNS policy programming will be handled.\nInline - BPF parses DNS response inline with DNS response packet\nprocessing. This guarantees the DNS rules reflect any change immediately.\nNoDelay - Felix does not introduce any delay to the packets. DNS rules may not have been programmed by the time\nthe first packet traverses the policy rules. Client applications need to handle reconnection attempts if initial\nconnection attempts fail. This may be problematic for some applications or for very low DNS TTLs.", + "DescriptionHTML": "

Specifies how DNS policy programming will be handled.\nInline - BPF parses DNS response inline with DNS response packet\nprocessing. This guarantees the DNS rules reflect any change immediately.\nNoDelay - Felix does not introduce any delay to the packets. DNS rules may not have been programmed by the time\nthe first packet traverses the policy rules. Client applications need to handle reconnection attempts if initial\nconnection attempts fail. This may be problematic for some applications or for very low DNS TTLs.

", + "UserEditable": true, + "GoType": "*v3.BPFDNSPolicyMode", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFDSROptoutCIDRs", + "NameEnvVar": "FELIX_BPFDSROptoutCIDRs", + "NameYAML": "bpfDSROptoutCIDRs", + "NameGoAPI": "BPFDSROptoutCIDRs", + "StringSchema": "Comma-delimited list of CIDRs", + "StringSchemaHTML": "Comma-delimited list of CIDRs", + "StringDefault": "", + "ParsedDefault": "[]", + "ParsedDefaultJSON": "null", + "ParsedType": "[]string", + "YAMLType": "array", + "YAMLSchema": "List of CIDRs: `[\"\", ...]`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of CIDRs: [\"<cidr>\", ...].", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A list of CIDRs which are excluded from DSR. That is, clients\nin those CIDRs will access service node ports as if BPFExternalServiceMode was set to\nTunnel.", + "DescriptionHTML": "

A list of CIDRs which are excluded from DSR. That is, clients\nin those CIDRs will access service node ports as if BPFExternalServiceMode was set to\nTunnel.

", + "UserEditable": true, + "GoType": "*[]string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFDataIfacePattern", + "NameEnvVar": "FELIX_BPFDataIfacePattern", + "NameYAML": "bpfDataIfacePattern", + "NameGoAPI": "BPFDataIfacePattern", + "StringSchema": "Regular expression", + "StringSchemaHTML": "Regular expression", + "StringDefault": "^((en|wl|ww|sl|ib)[Popsx].*|(eth|wlan|wwan|bond).*)", + "ParsedDefault": "^((en|wl|ww|sl|ib)[Popsx].*|(eth|wlan|wwan|bond).*)", + "ParsedDefaultJSON": "\"^((en|wl|ww|sl|ib)[Popsx].*|(eth|wlan|wwan|bond).*)\"", + "ParsedType": "*regexp.Regexp", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "^((en|wl|ww|sl|ib)[Popsx].*|(eth|wlan|wwan|bond).*)", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A regular expression that controls which interfaces Felix should attach BPF programs to\nin order to catch traffic to/from the network. This needs to match the interfaces that Calico workload traffic\nflows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the\ncluster. It should not match the workload interfaces (usually named cali...) or any other special device managed\nby Calico itself (e.g., tunnels).", + "DescriptionHTML": "

A regular expression that controls which interfaces Felix should attach BPF programs to\nin order to catch traffic to/from the network. This needs to match the interfaces that Calico workload traffic\nflows over as well as any interfaces that handle incoming traffic to nodeports and services from outside the\ncluster. It should not match the workload interfaces (usually named cali...) or any other special device managed\nby Calico itself (e.g., tunnels).

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFDisableGROForIfaces", + "NameEnvVar": "FELIX_BPFDisableGROForIfaces", + "NameYAML": "bpfDisableGROForIfaces", + "NameGoAPI": "BPFDisableGROForIfaces", + "StringSchema": "Regular expression", + "StringSchemaHTML": "Regular expression", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "null", + "ParsedType": "*regexp.Regexp", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A regular expression that controls which interfaces Felix should disable the\nGeneric Receive Offload [GRO] option. It should not match the workload interfaces (usually named cali...).", + "DescriptionHTML": "

A regular expression that controls which interfaces Felix should disable the\nGeneric Receive Offload [GRO] option. It should not match the workload interfaces (usually named cali...).

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFDisableUnprivileged", + "NameEnvVar": "FELIX_BPFDisableUnprivileged", + "NameYAML": "bpfDisableUnprivileged", + "NameGoAPI": "BPFDisableUnprivileged", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "If enabled, Felix sets the kernel.unprivileged_bpf_disabled sysctl to disable\nunprivileged use of BPF. This ensures that unprivileged users cannot access Calico's BPF maps and\ncannot insert their own BPF programs to interfere with Calico's.", + "DescriptionHTML": "

If enabled, Felix sets the kernel.unprivileged_bpf_disabled sysctl to disable\nunprivileged use of BPF. This ensures that unprivileged users cannot access Calico's BPF maps and\ncannot insert their own BPF programs to interfere with Calico's.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFEnabled", + "NameEnvVar": "FELIX_BPFEnabled", + "NameYAML": "bpfEnabled", + "NameGoAPI": "BPFEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "If enabled Felix will use the BPF dataplane.", + "DescriptionHTML": "

If enabled Felix will use the BPF dataplane.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFEnforceRPF", + "NameEnvVar": "FELIX_BPFEnforceRPF", + "NameYAML": "bpfEnforceRPF", + "NameGoAPI": "BPFEnforceRPF", + "StringSchema": "One of: `Disabled`, `Loose`, `Strict`", + "StringSchemaHTML": "One of: Disabled, Loose, Strict", + "StringDefault": "Loose", + "ParsedDefault": "Loose", + "ParsedDefaultJSON": "\"Loose\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `Loose`, `Strict`.", + "YAMLEnumValues": [ + "Disabled", + "Loose", + "Strict" + ], + "YAMLSchemaHTML": "One of: Disabled, Loose, Strict.", + "YAMLDefault": "Loose", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Enforce strict RPF on all host interfaces with BPF programs regardless of\nwhat is the per-interfaces or global setting. Possible values are Disabled, Strict\nor Loose.", + "DescriptionHTML": "

Enforce strict RPF on all host interfaces with BPF programs regardless of\nwhat is the per-interfaces or global setting. Possible values are Disabled, Strict\nor Loose.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFExcludeCIDRsFromNAT", + "NameEnvVar": "FELIX_BPFExcludeCIDRsFromNAT", + "NameYAML": "bpfExcludeCIDRsFromNAT", + "NameGoAPI": "BPFExcludeCIDRsFromNAT", + "StringSchema": "Comma-delimited list of CIDRs", + "StringSchemaHTML": "Comma-delimited list of CIDRs", + "StringDefault": "", + "ParsedDefault": "[]", + "ParsedDefaultJSON": "null", + "ParsedType": "[]string", + "YAMLType": "array", + "YAMLSchema": "List of CIDRs: `[\"\", ...]`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of CIDRs: [\"<cidr>\", ...].", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A list of CIDRs that are to be excluded from NAT\nresolution so that host can handle them. A typical usecase is node local\nDNS cache.", + "DescriptionHTML": "

A list of CIDRs that are to be excluded from NAT\nresolution so that host can handle them. A typical usecase is node local\nDNS cache.

", + "UserEditable": true, + "GoType": "*[]string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFExportBufferSizeMB", + "NameEnvVar": "FELIX_BPFExportBufferSizeMB", + "NameYAML": "bpfExportBufferSizeMB", + "NameGoAPI": "BPFExportBufferSizeMB", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "1", + "ParsedDefault": "1", + "ParsedDefaultJSON": "1", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "1", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "In BPF mode, controls the buffer size used for sending BPF events to felix.", + "DescriptionHTML": "

In BPF mode, controls the buffer size used for sending BPF events to felix.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFExtToServiceConnmark", + "NameEnvVar": "FELIX_BPFExtToServiceConnmark", + "NameYAML": "bpfExtToServiceConnmark", + "NameGoAPI": "BPFExtToServiceConnmark", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "0", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "In BPF mode, controls a 32bit mark that is set on connections from an\nexternal client to a local service. This mark allows us to control how packets of that\nconnection are routed within the host and how is routing interpreted by RPF check.", + "DescriptionHTML": "

In BPF mode, controls a 32bit mark that is set on connections from an\nexternal client to a local service. This mark allows us to control how packets of that\nconnection are routed within the host and how is routing interpreted by RPF check.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFExternalServiceMode", + "NameEnvVar": "FELIX_BPFExternalServiceMode", + "NameYAML": "bpfExternalServiceMode", + "NameGoAPI": "BPFExternalServiceMode", + "StringSchema": "One of: `dsr`, `tunnel`", + "StringSchemaHTML": "One of: dsr, tunnel", + "StringDefault": "tunnel", + "ParsedDefault": "tunnel", + "ParsedDefaultJSON": "\"tunnel\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `DSR`, `Tunnel`.", + "YAMLEnumValues": [ + "DSR", + "Tunnel" + ], + "YAMLSchemaHTML": "One of: DSR, Tunnel.", + "YAMLDefault": "Tunnel", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "In BPF mode, controls how connections from outside the cluster to services (node ports\nand cluster IPs) are forwarded to remote workloads. If set to \"Tunnel\" then both request and response traffic\nis tunneled to the remote node. If set to \"DSR\", the request traffic is tunneled but the response traffic\nis sent directly from the remote node. In \"DSR\" mode, the remote node appears to use the IP of the ingress\nnode; this requires a permissive L2 network.", + "DescriptionHTML": "

In BPF mode, controls how connections from outside the cluster to services (node ports\nand cluster IPs) are forwarded to remote workloads. If set to \"Tunnel\" then both request and response traffic\nis tunneled to the remote node. If set to \"DSR\", the request traffic is tunneled but the response traffic\nis sent directly from the remote node. In \"DSR\" mode, the remote node appears to use the IP of the ingress\nnode; this requires a permissive L2 network.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFForceTrackPacketsFromIfaces", + "NameEnvVar": "FELIX_BPFForceTrackPacketsFromIfaces", + "NameYAML": "bpfForceTrackPacketsFromIfaces", + "NameGoAPI": "BPFForceTrackPacketsFromIfaces", + "StringSchema": "Comma-delimited list of strings, each matching the regex `^[a-zA-Z0-9:._+-]{1,15}$`", + "StringSchemaHTML": "Comma-delimited list of strings, each matching the regex ^[a-zA-Z0-9:._+-]{1,15}$", + "StringDefault": "docker+", + "ParsedDefault": "[docker+]", + "ParsedDefaultJSON": "[\"docker+\"]", + "ParsedType": "[]string", + "YAMLType": "array", + "YAMLSchema": "List of interface names (may use `+` as a wildcard: `[\"\", ...]`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of interface names (may use + as a wildcard: [\"<name>\", ...].", + "YAMLDefault": "[\"docker+\"]", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "In BPF mode, forces traffic from these interfaces\nto skip Calico's iptables NOTRACK rule, allowing traffic from those interfaces to be\ntracked by Linux conntrack. Should only be used for interfaces that are not used for\nthe Calico fabric. For example, a docker bridge device for non-Calico-networked\ncontainers.", + "DescriptionHTML": "

In BPF mode, forces traffic from these interfaces\nto skip Calico's iptables NOTRACK rule, allowing traffic from those interfaces to be\ntracked by Linux conntrack. Should only be used for interfaces that are not used for\nthe Calico fabric. For example, a docker bridge device for non-Calico-networked\ncontainers.

", + "UserEditable": true, + "GoType": "*[]string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFHostConntrackBypass", + "NameEnvVar": "FELIX_BPFHostConntrackBypass", + "NameYAML": "bpfHostConntrackBypass", + "NameGoAPI": "BPFHostConntrackBypass", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether to bypass Linux conntrack in BPF mode for\nworkloads and services.", + "DescriptionHTML": "

Controls whether to bypass Linux conntrack in BPF mode for\nworkloads and services.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFHostNetworkedNATWithoutCTLB", + "NameEnvVar": "FELIX_BPFHostNetworkedNATWithoutCTLB", + "NameYAML": "bpfHostNetworkedNATWithoutCTLB", + "NameGoAPI": "BPFHostNetworkedNATWithoutCTLB", + "StringSchema": "One of: `Disabled`, `Enabled`", + "StringSchemaHTML": "One of: Disabled, Enabled", + "StringDefault": "Enabled", + "ParsedDefault": "Enabled", + "ParsedDefaultJSON": "\"Enabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Disabled\"`, `\"Enabled\"`.", + "YAMLEnumValues": [ + "Disabled", + "Enabled" + ], + "YAMLSchemaHTML": "One of: \"Disabled\", \"Enabled\".", + "YAMLDefault": "Enabled", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "When in BPF mode, controls whether Felix does a NAT without CTLB. This along with BPFConnectTimeLoadBalancing\ndetermines the CTLB behavior.", + "DescriptionHTML": "

When in BPF mode, controls whether Felix does a NAT without CTLB. This along with BPFConnectTimeLoadBalancing\ndetermines the CTLB behavior.

", + "UserEditable": true, + "GoType": "*v3.BPFHostNetworkedNATType", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFIPFragTimeout", + "NameEnvVar": "FELIX_BPFIPFragTimeout", + "NameYAML": "bpfIPFragTimeout", + "NameGoAPI": "BPFIPFragTimeout", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "0", + "ParsedDefault": "0s", + "ParsedDefaultJSON": "0", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "In BPF mode, controls the timeout for IP fragment reassembly.\nThis is the maximum time that the BPF dataplane will wait for all fragments of a\nfragmented IP packet to arrive before discarding them. If left unset, the value\nis read from the Linux kernel sysctl net.ipv4.ipfrag_time (which defaults to 30\nseconds).", + "DescriptionHTML": "

In BPF mode, controls the timeout for IP fragment reassembly.\nThis is the maximum time that the BPF dataplane will wait for all fragments of a\nfragmented IP packet to arrive before discarding them. If left unset, the value\nis read from the Linux kernel sysctl net.ipv4.ipfrag_time (which defaults to 30\nseconds).

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFIPFragmentReassemblyEnabled", + "NameEnvVar": "FELIX_BPFIPFragmentReassemblyEnabled", + "NameYAML": "bpfIPFragmentReassemblyEnabled", + "NameGoAPI": "BPFIPFragmentReassemblyEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix loads the BPF program that\nreassembles out-of-order IP fragments from external networks. This program requires\na kernel newer than 5.10. When enabled (the default) and the program fails to load,\nFelix reports not-ready until the user sets this to false. When false, fragmented\npackets from external sources are dropped.", + "DescriptionHTML": "

Controls whether Felix loads the BPF program that\nreassembles out-of-order IP fragments from external networks. This program requires\na kernel newer than 5.10. When enabled (the default) and the program fails to load,\nFelix reports not-ready until the user sets this to false. When false, fragmented\npackets from external sources are dropped.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFJITHardening", + "NameEnvVar": "FELIX_BPFJITHardening", + "NameYAML": "bpfJITHardening", + "NameGoAPI": "BPFJITHardening", + "StringSchema": "One of: `Auto`, `Strict`", + "StringSchemaHTML": "One of: Auto, Strict", + "StringDefault": "Auto", + "ParsedDefault": "Auto", + "ParsedDefaultJSON": "\"Auto\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "Auto", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls BPF JIT hardening. When set to \"Auto\", Felix will set JIT hardening to 1\nif it detects the current value is 2 (strict mode that hurts performance). When set to \"Strict\",\nFelix will not modify the JIT hardening setting.", + "DescriptionHTML": "

Controls BPF JIT hardening. When set to \"Auto\", Felix will set JIT hardening to 1\nif it detects the current value is 2 (strict mode that hurts performance). When set to \"Strict\",\nFelix will not modify the JIT hardening setting.

", + "UserEditable": true, + "GoType": "*v3.BPFJITHardeningType", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFKubeProxyHealthzPort", + "NameEnvVar": "FELIX_BPFKubeProxyHealthzPort", + "NameYAML": "bpfKubeProxyHealthzPort", + "NameGoAPI": "BPFKubeProxyHealthzPort", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "10256", + "ParsedDefault": "10256", + "ParsedDefaultJSON": "10256", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "10256", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "In BPF mode, controls the port that Felix's embedded kube-proxy health check server binds to.\nThe health check server is used by external load balancers to determine if this node should receive traffic.\nSet to 0 to disable the health check server.", + "DescriptionHTML": "

In BPF mode, controls the port that Felix's embedded kube-proxy health check server binds to.\nThe health check server is used by external load balancers to determine if this node should receive traffic.\nSet to 0 to disable the health check server.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFKubeProxyIptablesCleanupEnabled", + "NameEnvVar": "FELIX_BPFKubeProxyIptablesCleanupEnabled", + "NameYAML": "bpfKubeProxyIptablesCleanupEnabled", + "NameGoAPI": "BPFKubeProxyIptablesCleanupEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "If enabled in BPF mode, Felix will proactively clean up the upstream\nKubernetes kube-proxy's iptables chains. Should only be enabled if kube-proxy is not running.", + "DescriptionHTML": "

If enabled in BPF mode, Felix will proactively clean up the upstream\nKubernetes kube-proxy's iptables chains. Should only be enabled if kube-proxy is not running.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFKubeProxyMinSyncPeriod", + "NameEnvVar": "FELIX_BPFKubeProxyMinSyncPeriod", + "NameYAML": "bpfKubeProxyMinSyncPeriod", + "NameGoAPI": "BPFKubeProxyMinSyncPeriod", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "1", + "ParsedDefault": "1s", + "ParsedDefaultJSON": "1000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "In BPF mode, controls the minimum time between updates to the dataplane for Felix's\nembedded kube-proxy. Lower values give reduced set-up latency. Higher values reduce Felix CPU usage by\nbatching up more work.", + "DescriptionHTML": "

In BPF mode, controls the minimum time between updates to the dataplane for Felix's\nembedded kube-proxy. Lower values give reduced set-up latency. Higher values reduce Felix CPU usage by\nbatching up more work.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFL3IfacePattern", + "NameEnvVar": "FELIX_BPFL3IfacePattern", + "NameYAML": "bpfL3IfacePattern", + "NameGoAPI": "BPFL3IfacePattern", + "StringSchema": "Regular expression", + "StringSchemaHTML": "Regular expression", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "null", + "ParsedType": "*regexp.Regexp", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A regular expression that allows to list tunnel devices like wireguard or vxlan (i.e., L3 devices)\nin addition to BPFDataIfacePattern. That is, tunnel interfaces not created by Calico, that Calico workload traffic flows\nover as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster.", + "DescriptionHTML": "

A regular expression that allows to list tunnel devices like wireguard or vxlan (i.e., L3 devices)\nin addition to BPFDataIfacePattern. That is, tunnel interfaces not created by Calico, that Calico workload traffic flows\nover as well as any interfaces that handle incoming traffic to nodeports and services from outside the cluster.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFLogFilters", + "NameEnvVar": "FELIX_BPFLogFilters", + "NameYAML": "bpfLogFilters", + "NameGoAPI": "BPFLogFilters", + "StringSchema": "Comma-delimited list of key=value pairs", + "StringSchemaHTML": "Comma-delimited list of key=value pairs", + "StringDefault": "", + "ParsedDefault": "map[]", + "ParsedDefaultJSON": "null", + "ParsedType": "map[string]string", + "YAMLType": "object", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A map of key=values where the value is\na pcap filter expression and the key is an interface name with 'all'\ndenoting all interfaces, 'weps' all workload endpoints and 'heps' all host\nendpoints.\n\nWhen specified as an env var, it accepts a comma-separated list of\nkey=values.", + "DescriptionHTML": "

A map of key=values where the value is\na pcap filter expression and the key is an interface name with 'all'\ndenoting all interfaces, 'weps' all workload endpoints and 'heps' all host\nendpoints.

\n

When specified as an env var, it accepts a comma-separated list of\nkey=values.

", + "UserEditable": true, + "GoType": "*map[string]string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFLogLevel", + "NameEnvVar": "FELIX_BPFLogLevel", + "NameYAML": "bpfLogLevel", + "NameGoAPI": "BPFLogLevel", + "StringSchema": "One of: `debug`, `info`, `off`", + "StringSchemaHTML": "One of: debug, info, off", + "StringDefault": "off", + "ParsedDefault": "off", + "ParsedDefaultJSON": "\"off\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Debug`, `Info`, `Off`.", + "YAMLEnumValues": [ + "Debug", + "Info", + "Off" + ], + "YAMLSchemaHTML": "One of: Debug, Info, Off.", + "YAMLDefault": "Off", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the log level of the BPF programs when in BPF dataplane mode. One of \"Off\", \"Info\", or\n\"Debug\". The logs are emitted to the BPF trace pipe, accessible with the command `tc exec bpf debug`.\n.", + "DescriptionHTML": "

Controls the log level of the BPF programs when in BPF dataplane mode. One of \"Off\", \"Info\", or\n\"Debug\". The logs are emitted to the BPF trace pipe, accessible with the command tc exec bpf debug.\n.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMaglevMaxEndpointsPerService", + "NameEnvVar": "FELIX_BPFMaglevMaxEndpointsPerService", + "NameYAML": "bpfMaglevMaxEndpointsPerService", + "NameGoAPI": "BPFMaglevMaxEndpointsPerService", + "StringSchema": "Integer: [1,3000]", + "StringSchemaHTML": "Integer: [1,3000]", + "StringDefault": "100", + "ParsedDefault": "100", + "ParsedDefaultJSON": "100", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [1,3000]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [1,3000]", + "YAMLDefault": "100", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The maximum number of endpoints\nexpected to be part of a single Maglev-enabled service.\n\nInfluences the size of the per-service Maglev lookup-tables generated by Felix\nand thus the amount of memory reserved.", + "DescriptionHTML": "

The maximum number of endpoints\nexpected to be part of a single Maglev-enabled service.

\n

Influences the size of the per-service Maglev lookup-tables generated by Felix\nand thus the amount of memory reserved.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMaglevMaxServices", + "NameEnvVar": "FELIX_BPFMaglevMaxServices", + "NameYAML": "bpfMaglevMaxServices", + "NameGoAPI": "BPFMaglevMaxServices", + "StringSchema": "Integer: [1,3000]", + "StringSchemaHTML": "Integer: [1,3000]", + "StringDefault": "100", + "ParsedDefault": "100", + "ParsedDefaultJSON": "100", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [1,3000]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [1,3000]", + "YAMLDefault": "100", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The maximum number of expected Maglev-enabled\nservices that Felix will allocate lookup-tables for.", + "DescriptionHTML": "

The maximum number of expected Maglev-enabled\nservices that Felix will allocate lookup-tables for.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeConntrack", + "NameEnvVar": "FELIX_BPFMapSizeConntrack", + "NameYAML": "bpfMapSizeConntrack", + "NameGoAPI": "BPFMapSizeConntrack", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "512000", + "ParsedDefault": "512000", + "ParsedDefaultJSON": "512000", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "512000", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size for the conntrack map. This map must be large enough to hold\nan entry for each active connection. Warning: changing the size of the conntrack map can cause disruption.", + "DescriptionHTML": "

Sets the size for the conntrack map. This map must be large enough to hold\nan entry for each active connection. Warning: changing the size of the conntrack map can cause disruption.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeConntrackCleanupQueue", + "NameEnvVar": "FELIX_BPFMapSizeConntrackCleanupQueue", + "NameYAML": "bpfMapSizeConntrackCleanupQueue", + "NameGoAPI": "BPFMapSizeConntrackCleanupQueue", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "100000", + "ParsedDefault": "100000", + "ParsedDefaultJSON": "100000", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "100000", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size for the map used to hold NAT conntrack entries that are queued\nfor cleanup. This should be big enough to hold all the NAT entries that expire within one cleanup interval.", + "DescriptionHTML": "

Sets the size for the map used to hold NAT conntrack entries that are queued\nfor cleanup. This should be big enough to hold all the NAT entries that expire within one cleanup interval.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeConntrackScaling", + "NameEnvVar": "FELIX_BPFMapSizeConntrackScaling", + "NameYAML": "bpfMapSizeConntrackScaling", + "NameGoAPI": "BPFMapSizeConntrackScaling", + "StringSchema": "One of: `Disabled`, `DoubleIfFull`", + "StringSchemaHTML": "One of: Disabled, DoubleIfFull", + "StringDefault": "DoubleIfFull", + "ParsedDefault": "DoubleIfFull", + "ParsedDefaultJSON": "\"DoubleIfFull\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `DoubleIfFull`.", + "YAMLEnumValues": [ + "Disabled", + "DoubleIfFull" + ], + "YAMLSchemaHTML": "One of: Disabled, DoubleIfFull.", + "YAMLDefault": "DoubleIfFull", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether and how we scale the conntrack map size depending\non its usage. 'Disabled' make the size stay at the default or whatever is set by\nBPFMapSizeConntrack*. 'DoubleIfFull' doubles the size when the map is pretty much full even\nafter cleanups.", + "DescriptionHTML": "

Controls whether and how we scale the conntrack map size depending\non its usage. 'Disabled' make the size stay at the default or whatever is set by\nBPFMapSizeConntrack*. 'DoubleIfFull' doubles the size when the map is pretty much full even\nafter cleanups.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeIPSets", + "NameEnvVar": "FELIX_BPFMapSizeIPSets", + "NameYAML": "bpfMapSizeIPSets", + "NameGoAPI": "BPFMapSizeIPSets", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "1048576", + "ParsedDefault": "1048576", + "ParsedDefaultJSON": "1048576", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "1048576", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size for ipsets map. The IP sets map must be large enough to hold an entry\nfor each endpoint matched by every selector in the source/destination matches in network policy. Selectors\nsuch as \"all()\" can result in large numbers of entries (one entry per endpoint in that case).", + "DescriptionHTML": "

Sets the size for ipsets map. The IP sets map must be large enough to hold an entry\nfor each endpoint matched by every selector in the source/destination matches in network policy. Selectors\nsuch as \"all()\" can result in large numbers of entries (one entry per endpoint in that case).

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeIfState", + "NameEnvVar": "FELIX_BPFMapSizeIfState", + "NameYAML": "bpfMapSizeIfState", + "NameGoAPI": "BPFMapSizeIfState", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "1000", + "ParsedDefault": "1000", + "ParsedDefaultJSON": "1000", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "1000", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size for ifstate map. The ifstate map must be large enough to hold an entry\nfor each device (host + workloads) on a host.", + "DescriptionHTML": "

Sets the size for ifstate map. The ifstate map must be large enough to hold an entry\nfor each device (host + workloads) on a host.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeNATAffinity", + "NameEnvVar": "FELIX_BPFMapSizeNATAffinity", + "NameYAML": "bpfMapSizeNATAffinity", + "NameGoAPI": "BPFMapSizeNATAffinity", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "65536", + "ParsedDefault": "65536", + "ParsedDefaultJSON": "65536", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "65536", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size of the BPF map that stores the affinity of a connection (for services that\nenable that feature.", + "DescriptionHTML": "

Sets the size of the BPF map that stores the affinity of a connection (for services that\nenable that feature.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeNATBackend", + "NameEnvVar": "FELIX_BPFMapSizeNATBackend", + "NameYAML": "bpfMapSizeNATBackend", + "NameGoAPI": "BPFMapSizeNATBackend", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "262144", + "ParsedDefault": "262144", + "ParsedDefaultJSON": "262144", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "262144", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size for NAT back end map.\nThis is the total number of endpoints. This is mostly\nmore than the size of the number of services.", + "DescriptionHTML": "

Sets the size for NAT back end map.\nThis is the total number of endpoints. This is mostly\nmore than the size of the number of services.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeNATFrontend", + "NameEnvVar": "FELIX_BPFMapSizeNATFrontend", + "NameYAML": "bpfMapSizeNATFrontend", + "NameGoAPI": "BPFMapSizeNATFrontend", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "65536", + "ParsedDefault": "65536", + "ParsedDefaultJSON": "65536", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "65536", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size for NAT front end map.\nFrontendMap should be large enough to hold an entry for each nodeport,\nexternal IP and each port in each service.", + "DescriptionHTML": "

Sets the size for NAT front end map.\nFrontendMap should be large enough to hold an entry for each nodeport,\nexternal IP and each port in each service.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizePerCPUConntrack", + "NameEnvVar": "FELIX_BPFMapSizePerCPUConntrack", + "NameYAML": "bpfMapSizePerCpuConntrack", + "NameGoAPI": "BPFMapSizePerCPUConntrack", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "0", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Determines the size of conntrack map based on the number of CPUs. If set to a\nnon-zero value, overrides BPFMapSizeConntrack with `BPFMapSizePerCPUConntrack * (Number of CPUs)`.\nThis map must be large enough to hold an entry for each active connection. Warning: changing the size of the\nconntrack map can cause disruption.", + "DescriptionHTML": "

Determines the size of conntrack map based on the number of CPUs. If set to a\nnon-zero value, overrides BPFMapSizeConntrack with BPFMapSizePerCPUConntrack * (Number of CPUs).\nThis map must be large enough to hold an entry for each active connection. Warning: changing the size of the\nconntrack map can cause disruption.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFMapSizeRoute", + "NameEnvVar": "FELIX_BPFMapSizeRoute", + "NameYAML": "bpfMapSizeRoute", + "NameGoAPI": "BPFMapSizeRoute", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "262144", + "ParsedDefault": "262144", + "ParsedDefaultJSON": "262144", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "262144", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the size for the routes map. The routes map should be large enough\nto hold one entry per workload and a handful of entries per host (enough to cover its own IPs and\ntunnel IPs).", + "DescriptionHTML": "

Sets the size for the routes map. The routes map should be large enough\nto hold one entry per workload and a handful of entries per host (enough to cover its own IPs and\ntunnel IPs).

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFPSNATPorts", + "NameEnvVar": "FELIX_BPFPSNATPorts", + "NameYAML": "bpfPSNATPorts", + "NameGoAPI": "BPFPSNATPorts", + "StringSchema": "Port range: either a single number in [0,65535] or a range of numbers `n:m`", + "StringSchemaHTML": "Port range: either a single number in [0,65535] or a range of numbers n:m", + "StringDefault": "20000:29999", + "ParsedDefault": "20000:29999", + "ParsedDefaultJSON": "\"20000:29999\"", + "ParsedType": "numorstring.Port", + "YAMLType": "integer or string", + "YAMLSchema": "Port range: either an integer in [0,65535] or a string, representing a range, in format `n:m`", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Port range: either an integer in [0,65535] or a string, representing a range, in format n:m", + "YAMLDefault": "20000:29999", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the range from which we randomly pick a port if there is a source port\ncollision. This should be within the ephemeral range as defined by RFC 6056 (1024–65535) and\npreferably outside the ephemeral ranges used by common operating systems. Linux uses\n32768–60999, while others mostly use the IANA defined range 49152–65535. It is not necessarily\na problem if this range overlaps with the operating systems. Both ends of the range are\ninclusive.", + "DescriptionHTML": "

Sets the range from which we randomly pick a port if there is a source port\ncollision. This should be within the ephemeral range as defined by RFC 6056 (1024–65535) and\npreferably outside the ephemeral ranges used by common operating systems. Linux uses\n32768–60999, while others mostly use the IANA defined range 49152–65535. It is not necessarily\na problem if this range overlaps with the operating systems. Both ends of the range are\ninclusive.

", + "UserEditable": true, + "GoType": "*numorstring.Port", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFPolicyDebugEnabled", + "NameEnvVar": "FELIX_BPFPolicyDebugEnabled", + "NameYAML": "bpfPolicyDebugEnabled", + "NameGoAPI": "BPFPolicyDebugEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "When true, Felix records detailed information\nabout the BPF policy programs, which can be examined with the calico-bpf command-line tool.", + "DescriptionHTML": "

When true, Felix records detailed information\nabout the BPF policy programs, which can be examined with the calico-bpf command-line tool.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFProfiling", + "NameEnvVar": "FELIX_BPFProfiling", + "NameYAML": "bpfProfiling", + "NameGoAPI": "BPFProfiling", + "StringSchema": "One of: `Disabled`, `Enabled`", + "StringSchemaHTML": "One of: Disabled, Enabled", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Disabled\"`, `\"Enabled\"`.", + "YAMLEnumValues": [ + "Disabled", + "Enabled" + ], + "YAMLSchemaHTML": "One of: \"Disabled\", \"Enabled\".", + "YAMLDefault": "Disabled", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls profiling of BPF programs. At the monent, it can be\nDisabled or Enabled.", + "DescriptionHTML": "

Controls profiling of BPF programs. At the monent, it can be\nDisabled or Enabled.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: eBPF", + "GroupWithSortPrefix": "22 Dataplane: eBPF", + "NameConfigFile": "BPFRedirectToPeer", + "NameEnvVar": "FELIX_BPFRedirectToPeer", + "NameYAML": "bpfRedirectToPeer", + "NameGoAPI": "BPFRedirectToPeer", + "StringSchema": "One of: `Disabled`, `Enabled`", + "StringSchemaHTML": "One of: Disabled, Enabled", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Disabled\"`, `\"Enabled\"`.", + "YAMLEnumValues": [ + "Disabled", + "Enabled" + ], + "YAMLSchemaHTML": "One of: \"Disabled\", \"Enabled\".", + "YAMLDefault": "Disabled", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether traffic may be forwarded directly to the peer side of a workload’s device.\nNote that the legacy \"L2Only\" option is now deprecated and if set it is treated like \"Enabled\".\nSetting this option to \"Enabled\" allows direct redirection (including from L3 host devices such as IPIP tunnels or WireGuard),\nwhich can improve redirection performance but causes the redirected packets to bypass the host‑side ingress path.\nAs a result, packet‑capture tools on the host side of the workload device (for example, tcpdump) will not see that traffic.", + "DescriptionHTML": "

Controls whether traffic may be forwarded directly to the peer side of a workload’s device.\nNote that the legacy \"L2Only\" option is now deprecated and if set it is treated like \"Enabled\".\nSetting this option to \"Enabled\" allows direct redirection (including from L3 host devices such as IPIP tunnels or WireGuard),\nwhich can improve redirection performance but causes the redirected packets to bypass the host‑side ingress path.\nAs a result, packet‑capture tools on the host side of the workload device (for example, tcpdump) will not see that traffic.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Dataplane: Windows", + "Fields": [ + { + "Group": "Dataplane: Windows", + "GroupWithSortPrefix": "23 Dataplane: Windows", + "NameConfigFile": "WindowsDNSCacheFile", + "NameEnvVar": "FELIX_WindowsDNSCacheFile", + "NameYAML": "windowsDnsCacheFile", + "NameGoAPI": "WindowsDNSCacheFile", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "c:\\TigeraCalico\\felix-dns-cache.txt", + "ParsedDefault": "c:\\TigeraCalico\\felix-dns-cache.txt", + "ParsedDefaultJSON": "\"c:\\\\TigeraCalico\\\\felix-dns-cache.txt\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "c:\\TigeraCalico\\felix-dns-cache.txt", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The name of the file that Felix uses to preserve learnt DNS information when restarting. .", + "DescriptionHTML": "

The name of the file that Felix uses to preserve learnt DNS information when restarting. .

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Windows", + "GroupWithSortPrefix": "23 Dataplane: Windows", + "NameConfigFile": "WindowsDNSExtraTTL", + "NameEnvVar": "FELIX_WindowsDNSExtraTTL", + "NameYAML": "windowsDnsExtraTTL", + "NameGoAPI": "WindowsDNSExtraTTL", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "120", + "ParsedDefault": "2m0s", + "ParsedDefaultJSON": "120000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "2m0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Extra time to keep IPs and alias names that are learnt from DNS, in addition to each name\nor IP's advertised TTL. The default value is 120s which is same as the default value of\nServicePointManager.DnsRefreshTimeout on .net framework. .", + "DescriptionHTML": "

Extra time to keep IPs and alias names that are learnt from DNS, in addition to each name\nor IP's advertised TTL. The default value is 120s which is same as the default value of\nServicePointManager.DnsRefreshTimeout on .net framework. .

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Windows", + "GroupWithSortPrefix": "23 Dataplane: Windows", + "NameConfigFile": "WindowsFlowLogsFileDirectory", + "NameEnvVar": "FELIX_WindowsFlowLogsFileDirectory", + "NameYAML": "windowsFlowLogsFileDirectory", + "NameGoAPI": "WindowsFlowLogsFileDirectory", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "c:\\TigeraCalico\\flowlogs", + "ParsedDefault": "c:\\TigeraCalico\\flowlogs", + "ParsedDefaultJSON": "\"c:\\\\TigeraCalico\\\\flowlogs\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "c:\\TigeraCalico\\flowlogs", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the directory where flow logs files are stored on Windows nodes. .", + "DescriptionHTML": "

Sets the directory where flow logs files are stored on Windows nodes. .

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Windows", + "GroupWithSortPrefix": "23 Dataplane: Windows", + "NameConfigFile": "WindowsFlowLogsPositionFilePath", + "NameEnvVar": "FELIX_WindowsFlowLogsPositionFilePath", + "NameYAML": "windowsFlowLogsPositionFilePath", + "NameGoAPI": "WindowsFlowLogsPositionFilePath", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "c:\\TigeraCalico\\flowlogs\\flows.log.pos", + "ParsedDefault": "c:\\TigeraCalico\\flowlogs\\flows.log.pos", + "ParsedDefaultJSON": "\"c:\\\\TigeraCalico\\\\flowlogs\\\\flows.log.pos\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "c:\\TigeraCalico\\flowlogs\\flows.log.pos", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to specify the position of the external pipeline that reads flow logs on Windows nodes.\n.\nThis parameter only takes effect when FlowLogsDynamicAggregationEnabled is set to true.", + "DescriptionHTML": "

Used to specify the position of the external pipeline that reads flow logs on Windows nodes.\n.\nThis parameter only takes effect when FlowLogsDynamicAggregationEnabled is set to true.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Windows", + "GroupWithSortPrefix": "23 Dataplane: Windows", + "NameConfigFile": "WindowsManageFirewallRules", + "NameEnvVar": "FELIX_WindowsManageFirewallRules", + "NameYAML": "windowsManageFirewallRules", + "NameGoAPI": "WindowsManageFirewallRules", + "StringSchema": "One of: `Disabled`, `Enabled`", + "StringSchemaHTML": "One of: Disabled, Enabled", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Disabled\"`, `\"Enabled\"`.", + "YAMLEnumValues": [ + "Disabled", + "Enabled" + ], + "YAMLSchemaHTML": "One of: \"Disabled\", \"Enabled\".", + "YAMLDefault": "Disabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures whether or not Felix will program Windows Firewall rules (to allow inbound access to its own metrics ports).", + "DescriptionHTML": "

Configures whether or not Felix will program Windows Firewall rules (to allow inbound access to its own metrics ports).

", + "UserEditable": true, + "GoType": "*v3.WindowsManageFirewallRulesMode", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Windows", + "GroupWithSortPrefix": "23 Dataplane: Windows", + "NameConfigFile": "WindowsNetworkName", + "NameEnvVar": "FELIX_WindowsNetworkName", + "NameYAML": "windowsNetworkName", + "NameGoAPI": "WindowsNetworkName", + "StringSchema": "Regular expression", + "StringSchemaHTML": "Regular expression", + "StringDefault": "(?i)calico.*", + "ParsedDefault": "(?i)calico.*", + "ParsedDefaultJSON": "\"(?i)calico.*\"", + "ParsedType": "*regexp.Regexp", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "(?i)calico.*", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies which Windows HNS networks Felix should operate on. The default is to match\nnetworks that start with \"calico\". Supports regular expression syntax.", + "DescriptionHTML": "

Specifies which Windows HNS networks Felix should operate on. The default is to match\nnetworks that start with \"calico\". Supports regular expression syntax.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: Windows", + "GroupWithSortPrefix": "23 Dataplane: Windows", + "NameConfigFile": "WindowsStatsDumpFilePath", + "NameEnvVar": "FELIX_WindowsStatsDumpFilePath", + "NameYAML": "windowsStatsDumpFilePath", + "NameGoAPI": "WindowsStatsDumpFilePath", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "c:\\TigeraCalico\\stats\\dump", + "ParsedDefault": "c:\\TigeraCalico\\stats\\dump", + "ParsedDefaultJSON": "\"c:\\\\TigeraCalico\\\\stats\\\\dump\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "c:\\TigeraCalico\\stats\\dump", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to specify the path of the stats dump file on Windows nodes.", + "DescriptionHTML": "

Used to specify the path of the stats dump file on Windows nodes.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Dataplane: OpenStack support", + "Fields": [ + { + "Group": "Dataplane: OpenStack support", + "GroupWithSortPrefix": "25 Dataplane: OpenStack support", + "NameConfigFile": "EndpointReportingDelaySecs", + "NameEnvVar": "FELIX_EndpointReportingDelaySecs", + "NameYAML": "endpointReportingDelay", + "NameGoAPI": "EndpointReportingDelay", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "1", + "ParsedDefault": "1s", + "ParsedDefaultJSON": "1000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "**Open source-only parameter**; OpenStack is not supported in Calico Enterprise/Cloud.\n\nThe delay before Felix reports endpoint status to the datastore. This is only used\nby the OpenStack integration.", + "DescriptionHTML": "

Open source-only parameter; OpenStack is not supported in Calico Enterprise/Cloud.

\n

The delay before Felix reports endpoint status to the datastore. This is only used\nby the OpenStack integration.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": true + }, + { + "Group": "Dataplane: OpenStack support", + "GroupWithSortPrefix": "25 Dataplane: OpenStack support", + "NameConfigFile": "EndpointReportingEnabled", + "NameEnvVar": "FELIX_EndpointReportingEnabled", + "NameYAML": "endpointReportingEnabled", + "NameGoAPI": "EndpointReportingEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "**Open source-only parameter**; OpenStack is not supported in Calico Enterprise/Cloud.\n\nControls whether Felix reports endpoint status to the datastore. This is only used\nby the OpenStack integration.", + "DescriptionHTML": "

Open source-only parameter; OpenStack is not supported in Calico Enterprise/Cloud.

\n

Controls whether Felix reports endpoint status to the datastore. This is only used\nby the OpenStack integration.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": true + }, + { + "Group": "Dataplane: OpenStack support", + "GroupWithSortPrefix": "25 Dataplane: OpenStack support", + "NameConfigFile": "MetadataAddr", + "NameEnvVar": "FELIX_MetadataAddr", + "NameYAML": "metadataAddr", + "NameGoAPI": "MetadataAddr", + "StringSchema": "String matching regex `^[a-zA-Z0-9_.-]+$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9_.-]+$", + "StringDefault": "127.0.0.1", + "ParsedDefault": "127.0.0.1", + "ParsedDefaultJSON": "\"127.0.0.1\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "127.0.0.1", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "**Open source-only parameter**; OpenStack is not supported in Calico Enterprise/Cloud.\n\nThe IP address or domain name of the server that can answer VM queries for\ncloud-init metadata. In OpenStack, this corresponds to the machine running nova-api (or in\nUbuntu, nova-api-metadata). A value of none (case-insensitive) means that Felix should not\nset up any NAT rule for the metadata path.", + "DescriptionHTML": "

Open source-only parameter; OpenStack is not supported in Calico Enterprise/Cloud.

\n

The IP address or domain name of the server that can answer VM queries for\ncloud-init metadata. In OpenStack, this corresponds to the machine running nova-api (or in\nUbuntu, nova-api-metadata). A value of none (case-insensitive) means that Felix should not\nset up any NAT rule for the metadata path.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": true + }, + { + "Group": "Dataplane: OpenStack support", + "GroupWithSortPrefix": "25 Dataplane: OpenStack support", + "NameConfigFile": "MetadataPort", + "NameEnvVar": "FELIX_MetadataPort", + "NameYAML": "metadataPort", + "NameGoAPI": "MetadataPort", + "StringSchema": "Integer: [0,65535]", + "StringSchemaHTML": "Integer: [0,65535]", + "StringDefault": "8775", + "ParsedDefault": "8775", + "ParsedDefaultJSON": "8775", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [0,65535]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [0,65535]", + "YAMLDefault": "8775", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "**Open source-only parameter**; OpenStack is not supported in Calico Enterprise/Cloud.\n\nThe port of the metadata server. This, combined with global.MetadataAddr (if\nnot 'None'), is used to set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort.\nIn most cases this should not need to be changed .", + "DescriptionHTML": "

Open source-only parameter; OpenStack is not supported in Calico Enterprise/Cloud.

\n

The port of the metadata server. This, combined with global.MetadataAddr (if\nnot 'None'), is used to set up a NAT rule, from 169.254.169.254:80 to MetadataAddr:MetadataPort.\nIn most cases this should not need to be changed .

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": true + }, + { + "Group": "Dataplane: OpenStack support", + "GroupWithSortPrefix": "25 Dataplane: OpenStack support", + "NameConfigFile": "OpenstackRegion", + "NameEnvVar": "FELIX_OpenstackRegion", + "NameYAML": "openstackRegion", + "NameGoAPI": "OpenstackRegion", + "StringSchema": "OpenStack region name (must be a valid DNS label)", + "StringSchemaHTML": "OpenStack region name (must be a valid DNS label)", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "**Open source-only parameter**; OpenStack is not supported in Calico Enterprise/Cloud.\n\nThe name of the region that a particular Felix belongs to. In a multi-region\nCalico/OpenStack deployment, this must be configured somehow for each Felix (here in the datamodel,\nor in felix.cfg or the environment on each compute node), and must match the [calico]\nopenstack_region value configured in neutron.conf on each node.", + "DescriptionHTML": "

Open source-only parameter; OpenStack is not supported in Calico Enterprise/Cloud.

\n

The name of the region that a particular Felix belongs to. In a multi-region\nCalico/OpenStack deployment, this must be configured somehow for each Felix (here in the datamodel,\nor in felix.cfg or the environment on each compute node), and must match the [calico]\nopenstack_region value configured in neutron.conf on each node.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": true + }, + { + "Group": "Dataplane: OpenStack support", + "GroupWithSortPrefix": "25 Dataplane: OpenStack support", + "NameConfigFile": "ReportingIntervalSecs", + "NameEnvVar": "FELIX_ReportingIntervalSecs", + "NameYAML": "reportingInterval", + "NameGoAPI": "ReportingInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "30", + "ParsedDefault": "30s", + "ParsedDefaultJSON": "30000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "**Open source-only parameter**; OpenStack is not supported in Calico Enterprise/Cloud.\n\nThe interval at which Felix reports its status into the datastore or 0 to disable.\nMust be non-zero in OpenStack deployments.", + "DescriptionHTML": "

Open source-only parameter; OpenStack is not supported in Calico Enterprise/Cloud.

\n

The interval at which Felix reports its status into the datastore or 0 to disable.\nMust be non-zero in OpenStack deployments.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": true + }, + { + "Group": "Dataplane: OpenStack support", + "GroupWithSortPrefix": "25 Dataplane: OpenStack support", + "NameConfigFile": "ReportingTTLSecs", + "NameEnvVar": "FELIX_ReportingTTLSecs", + "NameYAML": "reportingTTL", + "NameGoAPI": "ReportingTTL", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "90", + "ParsedDefault": "1m30s", + "ParsedDefaultJSON": "90000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1m30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "**Open source-only parameter**; OpenStack is not supported in Calico Enterprise/Cloud.\n\nThe time-to-live setting for process-wide status reports.", + "DescriptionHTML": "

Open source-only parameter; OpenStack is not supported in Calico Enterprise/Cloud.

\n

The time-to-live setting for process-wide status reports.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": true + } + ] + }, + { + "Name": "Dataplane: XDP acceleration for iptables dataplane", + "Fields": [ + { + "Group": "Dataplane: XDP acceleration for iptables dataplane", + "GroupWithSortPrefix": "25 Dataplane: XDP acceleration for iptables dataplane", + "NameConfigFile": "GenericXDPEnabled", + "NameEnvVar": "FELIX_GenericXDPEnabled", + "NameYAML": "genericXDPEnabled", + "NameGoAPI": "GenericXDPEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Enables Generic XDP so network cards that don't support XDP offload or driver\nmodes can use XDP. This is not recommended since it doesn't provide better performance than\niptables.", + "DescriptionHTML": "

Enables Generic XDP so network cards that don't support XDP offload or driver\nmodes can use XDP. This is not recommended since it doesn't provide better performance than\niptables.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: XDP acceleration for iptables dataplane", + "GroupWithSortPrefix": "25 Dataplane: XDP acceleration for iptables dataplane", + "NameConfigFile": "XDPEnabled", + "NameEnvVar": "FELIX_XDPEnabled", + "NameYAML": "xdpEnabled", + "NameGoAPI": "XDPEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Enables XDP acceleration for suitable untracked incoming deny rules.", + "DescriptionHTML": "

Enables XDP acceleration for suitable untracked incoming deny rules.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Dataplane: XDP acceleration for iptables dataplane", + "GroupWithSortPrefix": "25 Dataplane: XDP acceleration for iptables dataplane", + "NameConfigFile": "XDPRefreshInterval", + "NameEnvVar": "FELIX_XDPRefreshInterval", + "NameYAML": "xdpRefreshInterval", + "NameGoAPI": "XDPRefreshInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "90", + "ParsedDefault": "1m30s", + "ParsedDefaultJSON": "90000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1m30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The period at which Felix re-checks all XDP state to ensure that no\nother process has accidentally broken Calico's BPF maps or attached programs. Set to 0 to\ndisable XDP refresh.", + "DescriptionHTML": "

The period at which Felix re-checks all XDP state to ensure that no\nother process has accidentally broken Calico's BPF maps or attached programs. Set to 0 to\ndisable XDP refresh.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Overlay: VXLAN overlay", + "Fields": [ + { + "Group": "Overlay: VXLAN overlay", + "GroupWithSortPrefix": "31 Overlay: VXLAN overlay", + "NameConfigFile": "VXLANEnabled", + "NameEnvVar": "FELIX_VXLANEnabled", + "NameYAML": "vxlanEnabled", + "NameGoAPI": "VXLANEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "null", + "ParsedType": "*bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Overrides whether Felix should create the VXLAN tunnel device for IPv4 VXLAN networking.\nOptional as Felix determines this based on the existing IP pools.", + "DescriptionHTML": "

Overrides whether Felix should create the VXLAN tunnel device for IPv4 VXLAN networking.\nOptional as Felix determines this based on the existing IP pools.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: VXLAN overlay", + "GroupWithSortPrefix": "31 Overlay: VXLAN overlay", + "NameConfigFile": "VXLANMTU", + "NameEnvVar": "FELIX_VXLANMTU", + "NameYAML": "vxlanMTU", + "NameGoAPI": "VXLANMTU", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "0", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The MTU to set on the IPv4 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the\nMTU of the host's interfaces.", + "DescriptionHTML": "

The MTU to set on the IPv4 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the\nMTU of the host's interfaces.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: VXLAN overlay", + "GroupWithSortPrefix": "31 Overlay: VXLAN overlay", + "NameConfigFile": "VXLANMTUV6", + "NameEnvVar": "FELIX_VXLANMTUV6", + "NameYAML": "vxlanMTUV6", + "NameGoAPI": "VXLANMTUV6", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "0", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The MTU to set on the IPv6 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the\nMTU of the host's interfaces.", + "DescriptionHTML": "

The MTU to set on the IPv6 VXLAN tunnel device. Optional as Felix auto-detects the MTU based on the\nMTU of the host's interfaces.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: VXLAN overlay", + "GroupWithSortPrefix": "31 Overlay: VXLAN overlay", + "NameConfigFile": "VXLANPort", + "NameEnvVar": "FELIX_VXLANPort", + "NameYAML": "vxlanPort", + "NameGoAPI": "VXLANPort", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "4789", + "ParsedDefault": "4789", + "ParsedDefaultJSON": "4789", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "4789", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The UDP port number to use for VXLAN traffic.", + "DescriptionHTML": "

The UDP port number to use for VXLAN traffic.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: VXLAN overlay", + "GroupWithSortPrefix": "31 Overlay: VXLAN overlay", + "NameConfigFile": "VXLANVNI", + "NameEnvVar": "FELIX_VXLANVNI", + "NameYAML": "vxlanVNI", + "NameGoAPI": "VXLANVNI", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "4096", + "ParsedDefault": "4096", + "ParsedDefaultJSON": "4096", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "4096", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The VXLAN VNI to use for VXLAN traffic. You may need to change this if the default value is\nin use on your system.", + "DescriptionHTML": "

The VXLAN VNI to use for VXLAN traffic. You may need to change this if the default value is\nin use on your system.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Overlay: IP-in-IP", + "Fields": [ + { + "Group": "Overlay: IP-in-IP", + "GroupWithSortPrefix": "32 Overlay: IP-in-IP", + "NameConfigFile": "IpInIpEnabled", + "NameEnvVar": "FELIX_IpInIpEnabled", + "NameYAML": "ipipEnabled", + "NameGoAPI": "IPIPEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "null", + "ParsedType": "*bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Overrides whether Felix should configure an IPIP interface on the host. Optional as Felix\ndetermines this based on the existing IP pools.", + "DescriptionHTML": "

Overrides whether Felix should configure an IPIP interface on the host. Optional as Felix\ndetermines this based on the existing IP pools.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: IP-in-IP", + "GroupWithSortPrefix": "32 Overlay: IP-in-IP", + "NameConfigFile": "IpInIpMtu", + "NameEnvVar": "FELIX_IpInIpMtu", + "NameYAML": "ipipMTU", + "NameGoAPI": "IPIPMTU", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "0", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the MTU to set on the IPIP tunnel device. Optional as Felix auto-detects the MTU based on the\nMTU of the host's interfaces.", + "DescriptionHTML": "

Controls the MTU to set on the IPIP tunnel device. Optional as Felix auto-detects the MTU based on the\nMTU of the host's interfaces.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Overlay: Wireguard", + "Fields": [ + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardEnabled", + "NameEnvVar": "FELIX_WireguardEnabled", + "NameYAML": "wireguardEnabled", + "NameGoAPI": "WireguardEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Wireguard is enabled for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network).", + "DescriptionHTML": "

Controls whether Wireguard is enabled for IPv4 (encapsulating IPv4 traffic over an IPv4 underlay network).

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardEnabledV6", + "NameEnvVar": "FELIX_WireguardEnabledV6", + "NameYAML": "wireguardEnabledV6", + "NameGoAPI": "WireguardEnabledV6", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Wireguard is enabled for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network).", + "DescriptionHTML": "

Controls whether Wireguard is enabled for IPv6 (encapsulating IPv6 traffic over an IPv6 underlay network).

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardHostEncryptionEnabled", + "NameEnvVar": "FELIX_WireguardHostEncryptionEnabled", + "NameYAML": "wireguardHostEncryptionEnabled", + "NameGoAPI": "WireguardHostEncryptionEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Wireguard host-to-host encryption is enabled.", + "DescriptionHTML": "

Controls whether Wireguard host-to-host encryption is enabled.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardInterfaceName", + "NameEnvVar": "FELIX_WireguardInterfaceName", + "NameYAML": "wireguardInterfaceName", + "NameGoAPI": "WireguardInterfaceName", + "StringSchema": "String matching regex `^[a-zA-Z0-9:._+-]{1,15}$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9:._+-]{1,15}$", + "StringDefault": "wireguard.cali", + "ParsedDefault": "wireguard.cali", + "ParsedDefaultJSON": "\"wireguard.cali\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "wireguard.cali", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies the name to use for the IPv4 Wireguard interface.", + "DescriptionHTML": "

Specifies the name to use for the IPv4 Wireguard interface.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardInterfaceNameV6", + "NameEnvVar": "FELIX_WireguardInterfaceNameV6", + "NameYAML": "wireguardInterfaceNameV6", + "NameGoAPI": "WireguardInterfaceNameV6", + "StringSchema": "String matching regex `^[a-zA-Z0-9:._+-]{1,15}$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9:._+-]{1,15}$", + "StringDefault": "wg-v6.cali", + "ParsedDefault": "wg-v6.cali", + "ParsedDefaultJSON": "\"wg-v6.cali\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "wg-v6.cali", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies the name to use for the IPv6 Wireguard interface.", + "DescriptionHTML": "

Specifies the name to use for the IPv6 Wireguard interface.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardListeningPort", + "NameEnvVar": "FELIX_WireguardListeningPort", + "NameYAML": "wireguardListeningPort", + "NameGoAPI": "WireguardListeningPort", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "51820", + "ParsedDefault": "51820", + "ParsedDefaultJSON": "51820", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "51820", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the listening port used by IPv4 Wireguard.", + "DescriptionHTML": "

Controls the listening port used by IPv4 Wireguard.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardListeningPortV6", + "NameEnvVar": "FELIX_WireguardListeningPortV6", + "NameYAML": "wireguardListeningPortV6", + "NameGoAPI": "WireguardListeningPortV6", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "51821", + "ParsedDefault": "51821", + "ParsedDefaultJSON": "51821", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "51821", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the listening port used by IPv6 Wireguard.", + "DescriptionHTML": "

Controls the listening port used by IPv6 Wireguard.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardMTU", + "NameEnvVar": "FELIX_WireguardMTU", + "NameYAML": "wireguardMTU", + "NameGoAPI": "WireguardMTU", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "0", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the MTU on the IPv4 Wireguard interface. See Configuring MTU.", + "DescriptionHTML": "

Controls the MTU on the IPv4 Wireguard interface. See Configuring MTU.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardMTUV6", + "NameEnvVar": "FELIX_WireguardMTUV6", + "NameYAML": "wireguardMTUV6", + "NameGoAPI": "WireguardMTUV6", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "0", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the MTU on the IPv6 Wireguard interface. See Configuring MTU.", + "DescriptionHTML": "

Controls the MTU on the IPv6 Wireguard interface. See Configuring MTU.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardPersistentKeepAlive", + "NameEnvVar": "FELIX_WireguardPersistentKeepAlive", + "NameYAML": "wireguardKeepAlive", + "NameGoAPI": "WireguardPersistentKeepAlive", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "0", + "ParsedDefault": "0s", + "ParsedDefaultJSON": "0", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls Wireguard PersistentKeepalive option. Set 0 to disable.", + "DescriptionHTML": "

Controls Wireguard PersistentKeepalive option. Set 0 to disable.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardRoutingRulePriority", + "NameEnvVar": "FELIX_WireguardRoutingRulePriority", + "NameYAML": "wireguardRoutingRulePriority", + "NameGoAPI": "WireguardRoutingRulePriority", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "99", + "ParsedDefault": "99", + "ParsedDefaultJSON": "99", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "99", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the priority value to use for the Wireguard routing rule.", + "DescriptionHTML": "

Controls the priority value to use for the Wireguard routing rule.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: Wireguard", + "GroupWithSortPrefix": "33 Overlay: Wireguard", + "NameConfigFile": "WireguardThreadingEnabled", + "NameEnvVar": "FELIX_WireguardThreadingEnabled", + "NameYAML": "wireguardThreadingEnabled", + "NameGoAPI": "WireguardThreadingEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Wireguard has Threaded NAPI enabled. \nThis increases the maximum number of packets a Wireguard interface can process.\nConsider threaded NAPI only if you have high packets per second workloads that are causing dropping packets due to a saturated `softirq` CPU core.\nThere is a [known issue](https://lore.kernel.org/netdev/CALrw=nEoT2emQ0OAYCjM1d_6Xe_kNLSZ6dhjb5FxrLFYh4kozA@mail.gmail.com/T/) with this setting\nthat may cause NAPI to get stuck holding the global `rtnl_mutex` when a peer is removed.\nWorkaround: Make sure your Linux kernel [includes this patch](https://github.com/torvalds/linux/commit/56364c910691f6d10ba88c964c9041b9ab777bd6) to unwedge NAPI.", + "DescriptionHTML": "

Controls whether Wireguard has Threaded NAPI enabled. \nThis increases the maximum number of packets a Wireguard interface can process.\nConsider threaded NAPI only if you have high packets per second workloads that are causing dropping packets due to a saturated softirq CPU core.\nThere is a [known issue](https://lore.kernel.org/netdev/CALrw=nEoT2emQ0OAYCjM1d_6Xe_kNLSZ6dhjb5FxrLFYh4kozA@mail.gmail.com/T/) with this setting\nthat may cause NAPI to get stuck holding the global rtnl_mutex when a peer is removed.\nWorkaround: Make sure your Linux kernel [includes this patch](https://github.com/torvalds/linux/commit/56364c910691f6d10ba88c964c9041b9ab777bd6) to unwedge NAPI.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Overlay: IPSec", + "Fields": [ + { + "Group": "Overlay: IPSec", + "GroupWithSortPrefix": "34 Overlay: IPSec", + "NameConfigFile": "IPSecAllowUnsecuredTraffic", + "NameEnvVar": "FELIX_IPSecAllowUnsecuredTraffic", + "NameYAML": "ipsecAllowUnsecuredTraffic", + "NameGoAPI": "IPSecAllowUnsecuredTraffic", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether non-IPsec traffic is allowed in addition to IPsec traffic. Enabling this\nnegates the anti-spoofing protections of IPsec but it is useful when migrating to/from IPsec.", + "DescriptionHTML": "

Controls whether non-IPsec traffic is allowed in addition to IPsec traffic. Enabling this\nnegates the anti-spoofing protections of IPsec but it is useful when migrating to/from IPsec.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: IPSec", + "GroupWithSortPrefix": "34 Overlay: IPSec", + "NameConfigFile": "IPSecESPAlgorithm", + "NameEnvVar": "FELIX_IPSecESPAlgorithm", + "NameYAML": "ipsecESPAlgorithm", + "NameGoAPI": "IPSecESPAlgorithm", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "aes128gcm16-ecp256", + "ParsedDefault": "aes128gcm16-ecp256", + "ParsedDefaultJSON": "\"aes128gcm16-ecp256\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "aes128gcm16-ecp256", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "IPSecESAlgorithm sets IPSec ESP algorithm. Default is NIST suite B recommendation.", + "DescriptionHTML": "

IPSecESAlgorithm sets IPSec ESP algorithm. Default is NIST suite B recommendation.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: IPSec", + "GroupWithSortPrefix": "34 Overlay: IPSec", + "NameConfigFile": "IPSecIKEAlgorithm", + "NameEnvVar": "FELIX_IPSecIKEAlgorithm", + "NameYAML": "ipsecIKEAlgorithm", + "NameGoAPI": "IPSecIKEAlgorithm", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "aes128gcm16-prfsha256-ecp256", + "ParsedDefault": "aes128gcm16-prfsha256-ecp256", + "ParsedDefaultJSON": "\"aes128gcm16-prfsha256-ecp256\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "aes128gcm16-prfsha256-ecp256", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets IPSec IKE algorithm. Default is NIST suite B recommendation.", + "DescriptionHTML": "

Sets IPSec IKE algorithm. Default is NIST suite B recommendation.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: IPSec", + "GroupWithSortPrefix": "34 Overlay: IPSec", + "NameConfigFile": "IPSecLogLevel", + "NameEnvVar": "FELIX_IPSecLogLevel", + "NameYAML": "ipsecLogLevel", + "NameGoAPI": "IPSecLogLevel", + "StringSchema": "One of: `DEBUG`, `INFO`, `NOTICE`, `VERBOSE`", + "StringSchemaHTML": "One of: DEBUG, INFO, NOTICE, VERBOSE", + "StringDefault": "INFO", + "ParsedDefault": "INFO", + "ParsedDefaultJSON": "\"INFO\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Debug`, `Info`, `None`, `Notice`, `Verbose`.", + "YAMLEnumValues": [ + "Debug", + "Info", + "None", + "Notice", + "Verbose" + ], + "YAMLSchemaHTML": "One of: Debug, Info, None, Notice, Verbose.", + "YAMLDefault": "Info", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls log level for IPSec components. Set to None for no logging.\nA generic log level terminology is used [None, Notice, Info, Debug, Verbose].", + "DescriptionHTML": "

Controls log level for IPSec components. Set to None for no logging.\nA generic log level terminology is used [None, Notice, Info, Debug, Verbose].

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: IPSec", + "GroupWithSortPrefix": "34 Overlay: IPSec", + "NameConfigFile": "IPSecMode", + "NameEnvVar": "FELIX_IPSecMode", + "NameYAML": "ipsecMode", + "NameGoAPI": "IPSecMode", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"\"`, `\"PSK\"`.", + "YAMLEnumValues": [ + "", + "PSK" + ], + "YAMLSchemaHTML": "One of: \"\", \"PSK\".", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls which mode IPSec is operating on.\nDefault value means IPSec is not enabled.", + "DescriptionHTML": "

Controls which mode IPSec is operating on.\nDefault value means IPSec is not enabled.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: IPSec", + "GroupWithSortPrefix": "34 Overlay: IPSec", + "NameConfigFile": "IPSecPSKFile", + "NameEnvVar": "FELIX_IPSecPSKFile", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "File contains PSK.", + "DescriptionHTML": "

File contains PSK.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Overlay: IPSec", + "GroupWithSortPrefix": "34 Overlay: IPSec", + "NameConfigFile": "IPSecPolicyRefreshInterval", + "NameEnvVar": "FELIX_IPSecPolicyRefreshInterval", + "NameYAML": "ipsecPolicyRefreshInterval", + "NameGoAPI": "IPSecPolicyRefreshInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "600", + "ParsedDefault": "10m0s", + "ParsedDefaultJSON": "600000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "10m0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The interval at which Felix will check the kernel’s IPsec policy tables and\nrepair any inconsistencies.", + "DescriptionHTML": "

The interval at which Felix will check the kernel’s IPsec policy tables and\nrepair any inconsistencies.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Flow logs: Prometheus reports", + "Fields": [ + { + "Group": "Flow logs: Prometheus reports", + "GroupWithSortPrefix": "40 Flow logs: Prometheus reports", + "NameConfigFile": "DeletedMetricsRetentionSecs", + "NameEnvVar": "FELIX_DeletedMetricsRetentionSecs", + "NameYAML": "deletedMetricsRetentionSecs", + "NameGoAPI": "DeletedMetricsRetentionSecs", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "30", + "ParsedDefault": "30s", + "ParsedDefaultJSON": "30000000000", + "ParsedType": "time.Duration", + "YAMLType": "integer", + "YAMLSchema": "Integer.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer.", + "YAMLDefault": "30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls how long metrics are retianed after the flow is gone.", + "DescriptionHTML": "

Controls how long metrics are retianed after the flow is gone.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: Prometheus reports", + "GroupWithSortPrefix": "40 Flow logs: Prometheus reports", + "NameConfigFile": "PrometheusReporterCAFile", + "NameEnvVar": "FELIX_PrometheusReporterCAFile", + "NameYAML": "prometheusReporterCAFile", + "NameGoAPI": "PrometheusReporterCAFile", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The path to the TLS CA file for the Prometheus per-flow metrics reporter.", + "DescriptionHTML": "

The path to the TLS CA file for the Prometheus per-flow metrics reporter.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: Prometheus reports", + "GroupWithSortPrefix": "40 Flow logs: Prometheus reports", + "NameConfigFile": "PrometheusReporterCertFile", + "NameEnvVar": "FELIX_PrometheusReporterCertFile", + "NameYAML": "prometheusReporterCertFile", + "NameGoAPI": "PrometheusReporterCertFile", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The path to the TLS certificate file for the Prometheus per-flow metrics reporter.", + "DescriptionHTML": "

The path to the TLS certificate file for the Prometheus per-flow metrics reporter.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: Prometheus reports", + "GroupWithSortPrefix": "40 Flow logs: Prometheus reports", + "NameConfigFile": "PrometheusReporterEnabled", + "NameEnvVar": "FELIX_PrometheusReporterEnabled", + "NameYAML": "prometheusReporterEnabled", + "NameGoAPI": "PrometheusReporterEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether the Prometheus per-flow metrics reporter is enabled. This is\nused to show real-time flow metrics in the UI.", + "DescriptionHTML": "

Controls whether the Prometheus per-flow metrics reporter is enabled. This is\nused to show real-time flow metrics in the UI.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: Prometheus reports", + "GroupWithSortPrefix": "40 Flow logs: Prometheus reports", + "NameConfigFile": "PrometheusReporterKeyFile", + "NameEnvVar": "FELIX_PrometheusReporterKeyFile", + "NameYAML": "prometheusReporterKeyFile", + "NameGoAPI": "PrometheusReporterKeyFile", + "StringSchema": "Path to file, which must exist", + "StringSchemaHTML": "Path to file, which must exist", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The path to the TLS private key file for the Prometheus per-flow metrics reporter.", + "DescriptionHTML": "

The path to the TLS private key file for the Prometheus per-flow metrics reporter.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: Prometheus reports", + "GroupWithSortPrefix": "40 Flow logs: Prometheus reports", + "NameConfigFile": "PrometheusReporterPort", + "NameEnvVar": "FELIX_PrometheusReporterPort", + "NameYAML": "prometheusReporterPort", + "NameGoAPI": "PrometheusReporterPort", + "StringSchema": "Integer: [0,65535]", + "StringSchemaHTML": "Integer: [0,65535]", + "StringDefault": "9092", + "ParsedDefault": "9092", + "ParsedDefaultJSON": "9092", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [0,65535]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [0,65535]", + "YAMLDefault": "9092", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The port that the Prometheus per-flow metrics reporter should bind to.", + "DescriptionHTML": "

The port that the Prometheus per-flow metrics reporter should bind to.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Flow logs: Syslog reports", + "Fields": [ + { + "Group": "Flow logs: Syslog reports", + "GroupWithSortPrefix": "40 Flow logs: Syslog reports", + "NameConfigFile": "SyslogReporterAddress", + "NameEnvVar": "FELIX_SyslogReporterAddress", + "NameYAML": "syslogReporterAddress", + "NameGoAPI": "SyslogReporterAddress", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The address to dial to when writing to Syslog. For TCP and UDP networks, the address has\nthe form \"host:port\". The host must be a literal IP address, or a host name that can be resolved to IP addresses.\nThe port must be a literal port number or a service name. For more, see: https://pkg.go.dev/net#Dial.", + "DescriptionHTML": "

The address to dial to when writing to Syslog. For TCP and UDP networks, the address has\nthe form \"host:port\". The host must be a literal IP address, or a host name that can be resolved to IP addresses.\nThe port must be a literal port number or a service name. For more, see: https://pkg.go.dev/net#Dial.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: Syslog reports", + "GroupWithSortPrefix": "40 Flow logs: Syslog reports", + "NameConfigFile": "SyslogReporterEnabled", + "NameEnvVar": "FELIX_SyslogReporterEnabled", + "NameYAML": "syslogReporterEnabled", + "NameGoAPI": "SyslogReporterEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Turns on the feature to write logs to Syslog. Please note that this can incur significant\ndisk space usage when running felix on non-cluster hosts.", + "DescriptionHTML": "

Turns on the feature to write logs to Syslog. Please note that this can incur significant\ndisk space usage when running felix on non-cluster hosts.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: Syslog reports", + "GroupWithSortPrefix": "40 Flow logs: Syslog reports", + "NameConfigFile": "SyslogReporterNetwork", + "NameEnvVar": "FELIX_SyslogReporterNetwork", + "NameYAML": "syslogReporterNetwork", + "NameGoAPI": "SyslogReporterNetwork", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The network to dial to when writing to Syslog. Known networks are \"tcp\", \"tcp4\"\n(IPv4-only), \"tcp6\" (IPv6-only), \"udp\", \"udp4\" (IPv4-only), \"udp6\" (IPv6-only), \"ip\", \"ip4\" (IPv4-only), \"ip6\"\n(IPv6-only), \"unix\", \"unixgram\" and \"unixpacket\". For more, see: https://pkg.go.dev/net#Dial.", + "DescriptionHTML": "

The network to dial to when writing to Syslog. Known networks are \"tcp\", \"tcp4\"\n(IPv4-only), \"tcp6\" (IPv6-only), \"udp\", \"udp4\" (IPv4-only), \"udp6\" (IPv6-only), \"ip\", \"ip4\" (IPv4-only), \"ip6\"\n(IPv6-only), \"unix\", \"unixgram\" and \"unixpacket\". For more, see: https://pkg.go.dev/net#Dial.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Flow logs: file reports", + "Fields": [ + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsAggregationThresholdBytes", + "NameEnvVar": "FELIX_FlowLogsAggregationThresholdBytes", + "NameYAML": "flowLogsAggregationThresholdBytes", + "NameGoAPI": "FlowLogsAggregationThresholdBytes", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "8192", + "ParsedDefault": "8192", + "ParsedDefaultJSON": "8192", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "8192", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used specify how far behind the external pipeline that reads flow logs can be. Default is 8192 bytes.\nThis parameter only takes effect when FlowLogsDynamicAggregationEnabled is set to true.", + "DescriptionHTML": "

Used specify how far behind the external pipeline that reads flow logs can be. Default is 8192 bytes.\nThis parameter only takes effect when FlowLogsDynamicAggregationEnabled is set to true.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsCollectProcessInfo", + "NameEnvVar": "FELIX_FlowLogsCollectProcessInfo", + "NameYAML": "flowLogsCollectProcessInfo", + "NameGoAPI": "FlowLogsCollectProcessInfo", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "If enabled Felix will load the kprobe BPF programs to collect process info.", + "DescriptionHTML": "

If enabled Felix will load the kprobe BPF programs to collect process info.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsCollectProcessPath", + "NameEnvVar": "FELIX_FlowLogsCollectProcessPath", + "NameYAML": "flowLogsCollectProcessPath", + "NameGoAPI": "FlowLogsCollectProcessPath", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "When FlowLogsCollectProcessPath and FlowLogsCollectProcessInfo are\nboth enabled, each flow log will include information about the process\nthat is sending or receiving the packets in that flow: the\n`process_name` field will contain the full path of the process\nexecutable, and the `process_args` field will have the arguments with\nwhich the executable was invoked. Process information will not be\nreported for connections which use raw sockets.", + "DescriptionHTML": "

When FlowLogsCollectProcessPath and FlowLogsCollectProcessInfo are\nboth enabled, each flow log will include information about the process\nthat is sending or receiving the packets in that flow: the\nprocess_name field will contain the full path of the process\nexecutable, and the process_args field will have the arguments with\nwhich the executable was invoked. Process information will not be\nreported for connections which use raw sockets.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsCollectTcpStats", + "NameEnvVar": "FELIX_FlowLogsCollectTcpStats", + "NameYAML": "flowLogsCollectTcpStats", + "NameGoAPI": "FlowLogsCollectTcpStats", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Enables flow logs reporting TCP socket stats.", + "DescriptionHTML": "

Enables flow logs reporting TCP socket stats.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsCollectorDebugTrace", + "NameEnvVar": "FELIX_FlowLogsCollectorDebugTrace", + "NameYAML": "flowLogsCollectorDebugTrace", + "NameGoAPI": "FlowLogsCollectorDebugTrace", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "When FlowLogsCollectorDebugTrace is set to true, enables the logs in the collector to be\nprinted in their entirety.", + "DescriptionHTML": "

When FlowLogsCollectorDebugTrace is set to true, enables the logs in the collector to be\nprinted in their entirety.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsDestDomainsByClient", + "NameEnvVar": "FELIX_FlowLogsDestDomainsByClient", + "NameYAML": "flowLogsDestDomainsByClient", + "NameGoAPI": "FlowLogsDestDomainsByClient", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to configure if the source IP is used in the mapping of top\nlevel destination domains.", + "DescriptionHTML": "

Used to configure if the source IP is used in the mapping of top\nlevel destination domains.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsDynamicAggregationEnabled", + "NameEnvVar": "FELIX_FlowLogsDynamicAggregationEnabled", + "NameYAML": "flowLogsDynamicAggregationEnabled", + "NameGoAPI": "FlowLogsDynamicAggregationEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to enable/disable dynamically changing aggregation levels. Default is false.", + "DescriptionHTML": "

Used to enable/disable dynamically changing aggregation levels. Default is false.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsEnableHostEndpoint", + "NameEnvVar": "FELIX_FlowLogsEnableHostEndpoint", + "NameYAML": "flowLogsEnableHostEndpoint", + "NameGoAPI": "FlowLogsEnableHostEndpoint", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Enables Flow logs reporting for HostEndpoints.", + "DescriptionHTML": "

Enables Flow logs reporting for HostEndpoints.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsEnableNetworkSets", + "NameEnvVar": "FELIX_FlowLogsEnableNetworkSets", + "NameYAML": "flowLogsEnableNetworkSets", + "NameGoAPI": "FlowLogsEnableNetworkSets", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Enables Flow logs reporting for GlobalNetworkSets.", + "DescriptionHTML": "

Enables Flow logs reporting for GlobalNetworkSets.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFileAggregationKindForAllowed", + "NameEnvVar": "FELIX_FlowLogsFileAggregationKindForAllowed", + "NameYAML": "flowLogsFileAggregationKindForAllowed", + "NameGoAPI": "FlowLogsFileAggregationKindForAllowed", + "StringSchema": "Integer: [0,3]", + "StringSchemaHTML": "Integer: [0,3]", + "StringDefault": "2", + "ParsedDefault": "2", + "ParsedDefaultJSON": "2", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "One of: `0`, `1`, `2`.", + "YAMLEnumValues": [ + "0", + "1", + "2" + ], + "YAMLSchemaHTML": "One of: 0, 1, 2.", + "YAMLDefault": "2", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to choose the type of aggregation for flow log entries created for\nallowed connections. .\nAccepted values are 0, 1 and 2.\n0 - No aggregation.\n1 - Source port based aggregation.\n2 - Pod prefix name based aggreagation.", + "DescriptionHTML": "

Used to choose the type of aggregation for flow log entries created for\nallowed connections. .\nAccepted values are 0, 1 and 2.\n0 - No aggregation.\n1 - Source port based aggregation.\n2 - Pod prefix name based aggreagation.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFileAggregationKindForDenied", + "NameEnvVar": "FELIX_FlowLogsFileAggregationKindForDenied", + "NameYAML": "flowLogsFileAggregationKindForDenied", + "NameGoAPI": "FlowLogsFileAggregationKindForDenied", + "StringSchema": "Integer: [0,3]", + "StringSchemaHTML": "Integer: [0,3]", + "StringDefault": "1", + "ParsedDefault": "1", + "ParsedDefaultJSON": "1", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "One of: `0`, `1`, `2`, `3`.", + "YAMLEnumValues": [ + "0", + "1", + "2", + "3" + ], + "YAMLSchemaHTML": "One of: 0, 1, 2, 3.", + "YAMLDefault": "1", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to choose the type of aggregation for flow log entries created for\ndenied connections. .\nAccepted values are 0, 1 and 2.\n0 - No aggregation.\n1 - Source port based aggregation.\n2 - Pod prefix name based aggregation.\n3 - No destination ports based aggregation.", + "DescriptionHTML": "

Used to choose the type of aggregation for flow log entries created for\ndenied connections. .\nAccepted values are 0, 1 and 2.\n0 - No aggregation.\n1 - Source port based aggregation.\n2 - Pod prefix name based aggregation.\n3 - No destination ports based aggregation.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFileDirectory", + "NameEnvVar": "FELIX_FlowLogsFileDirectory", + "NameYAML": "flowLogsFileDirectory", + "NameGoAPI": "FlowLogsFileDirectory", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "/var/log/calico/flowlogs", + "ParsedDefault": "/var/log/calico/flowlogs", + "ParsedDefaultJSON": "\"/var/log/calico/flowlogs\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "/var/log/calico/flowlogs", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the directory where flow logs files are stored.", + "DescriptionHTML": "

Sets the directory where flow logs files are stored.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFileDomainsLimit", + "NameEnvVar": "FELIX_FlowLogsFileDomainsLimit", + "NameYAML": "flowLogsFileDomainsLimit", + "NameGoAPI": "FlowLogsFileDomainsLimit", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "5", + "ParsedDefault": "5", + "ParsedDefaultJSON": "5", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "5", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to configure the number of (destination) domains to include in the flow log.\nThese are not included for workload or host endpoint destinations.", + "DescriptionHTML": "

Used to configure the number of (destination) domains to include in the flow log.\nThese are not included for workload or host endpoint destinations.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFileEnabled", + "NameEnvVar": "FELIX_FlowLogsFileEnabled", + "NameYAML": "flowLogsFileEnabled", + "NameGoAPI": "FlowLogsFileEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "When set to true, enables logging flow logs to a file. If false no flow logging to file will occur.", + "DescriptionHTML": "

When set to true, enables logging flow logs to a file. If false no flow logging to file will occur.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFileEnabledForAllowed", + "NameEnvVar": "FELIX_FlowLogsFileEnabledForAllowed", + "NameYAML": "flowLogsFileEnabledForAllowed", + "NameGoAPI": "FlowLogsFileEnabledForAllowed", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to enable/disable flow logs entries created for allowed connections. Default is true.\nThis parameter only takes effect when FlowLogsFileReporterEnabled is set to true.", + "DescriptionHTML": "

Used to enable/disable flow logs entries created for allowed connections. Default is true.\nThis parameter only takes effect when FlowLogsFileReporterEnabled is set to true.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFileEnabledForDenied", + "NameEnvVar": "FELIX_FlowLogsFileEnabledForDenied", + "NameYAML": "flowLogsFileEnabledForDenied", + "NameGoAPI": "FlowLogsFileEnabledForDenied", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to enable/disable flow logs entries created for denied flows. Default is true.\nThis parameter only takes effect when FlowLogsFileReporterEnabled is set to true.", + "DescriptionHTML": "

Used to enable/disable flow logs entries created for denied flows. Default is true.\nThis parameter only takes effect when FlowLogsFileReporterEnabled is set to true.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFileIncludeLabels", + "NameEnvVar": "FELIX_FlowLogsFileIncludeLabels", + "NameYAML": "flowLogsFileIncludeLabels", + "NameGoAPI": "FlowLogsFileIncludeLabels", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to configure if endpoint labels are included in a Flow log entry written to file.", + "DescriptionHTML": "

Used to configure if endpoint labels are included in a Flow log entry written to file.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFileIncludePolicies", + "NameEnvVar": "FELIX_FlowLogsFileIncludePolicies", + "NameYAML": "flowLogsFileIncludePolicies", + "NameGoAPI": "FlowLogsFileIncludePolicies", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to configure if policy information are included in a Flow log entry written to file.", + "DescriptionHTML": "

Used to configure if policy information are included in a Flow log entry written to file.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFileIncludeService", + "NameEnvVar": "FELIX_FlowLogsFileIncludeService", + "NameYAML": "flowLogsFileIncludeService", + "NameGoAPI": "FlowLogsFileIncludeService", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to configure if the destination service is included in a Flow log entry written to file.\nThe service information can only be included if the flow was explicitly determined to be directed at the service (e.g.\nwhen the pre-DNAT destination corresponds to the service ClusterIP and port).", + "DescriptionHTML": "

Used to configure if the destination service is included in a Flow log entry written to file.\nThe service information can only be included if the flow was explicitly determined to be directed at the service (e.g.\nwhen the pre-DNAT destination corresponds to the service ClusterIP and port).

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFileMaxFileSizeMB", + "NameEnvVar": "FELIX_FlowLogsFileMaxFileSizeMB", + "NameYAML": "flowLogsFileMaxFileSizeMB", + "NameGoAPI": "FlowLogsFileMaxFileSizeMB", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "100", + "ParsedDefault": "100", + "ParsedDefaultJSON": "100", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "100", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the max size in MB of flow logs files before rotation.", + "DescriptionHTML": "

Sets the max size in MB of flow logs files before rotation.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFileMaxFiles", + "NameEnvVar": "FELIX_FlowLogsFileMaxFiles", + "NameYAML": "flowLogsFileMaxFiles", + "NameGoAPI": "FlowLogsFileMaxFiles", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "5", + "ParsedDefault": "5", + "ParsedDefaultJSON": "5", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "5", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the number of log files to keep.", + "DescriptionHTML": "

Sets the number of log files to keep.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFileNatOutgoingPortLimit", + "NameEnvVar": "FELIX_FlowLogsFileNatOutgoingPortLimit", + "NameYAML": "flowLogsFileNatOutgoingPortLimit", + "NameGoAPI": "FlowLogsFileNatOutgoingPortLimit", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "3", + "ParsedDefault": "3", + "ParsedDefaultJSON": "3", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "3", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to specify the maximum number of distinct post SNAT ports that will appear\nin the flowLogs. Default value is 3.", + "DescriptionHTML": "

Used to specify the maximum number of distinct post SNAT ports that will appear\nin the flowLogs. Default value is 3.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFilePerFlowProcessArgsLimit", + "NameEnvVar": "FELIX_FlowLogsFilePerFlowProcessArgsLimit", + "NameYAML": "flowLogsFilePerFlowProcessArgsLimit", + "NameGoAPI": "FlowLogsFilePerFlowProcessArgsLimit", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "5", + "ParsedDefault": "5", + "ParsedDefaultJSON": "5", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "5", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to specify the maximum number of distinct process args that will appear in the flowLogs.\nDefault value is 5.", + "DescriptionHTML": "

Used to specify the maximum number of distinct process args that will appear in the flowLogs.\nDefault value is 5.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFilePerFlowProcessLimit", + "NameEnvVar": "FELIX_FlowLogsFilePerFlowProcessLimit", + "NameYAML": "flowLogsFilePerFlowProcessLimit", + "NameGoAPI": "FlowLogsFilePerFlowProcessLimit", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "2", + "ParsedDefault": "2", + "ParsedDefaultJSON": "2", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "2", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to specify the maximum number of flow log entries with distinct process information\nbeyond which process information will be aggregated.", + "DescriptionHTML": "

Used to specify the maximum number of flow log entries with distinct process information\nbeyond which process information will be aggregated.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsFlushInterval", + "NameEnvVar": "FELIX_FlowLogsFlushInterval", + "NameYAML": "flowLogsFlushInterval", + "NameGoAPI": "FlowLogsFlushInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "300", + "ParsedDefault": "5m0s", + "ParsedDefaultJSON": "300000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "5m0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures the interval at which Felix exports flow logs.", + "DescriptionHTML": "

Configures the interval at which Felix exports flow logs.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsGoldmaneServer", + "NameEnvVar": "FELIX_FlowLogsGoldmaneServer", + "NameYAML": "flowLogsGoldmaneServer", + "NameGoAPI": "FlowLogsGoldmaneServer", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "FlowLogGoldmaneServer is the flow server endpoint to which flow data should be published.", + "DescriptionHTML": "

FlowLogGoldmaneServer is the flow server endpoint to which flow data should be published.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsLocalReporter", + "NameEnvVar": "FELIX_FlowLogsLocalReporter", + "NameYAML": "flowLogsLocalReporter", + "NameGoAPI": "FlowLogsLocalReporter", + "StringSchema": "One of: `Disabled`, `Enabled`", + "StringSchemaHTML": "One of: Disabled, Enabled", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Disabled\"`, `\"Enabled\"`.", + "YAMLEnumValues": [ + "Disabled", + "Enabled" + ], + "YAMLSchemaHTML": "One of: \"Disabled\", \"Enabled\".", + "YAMLDefault": "Disabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures local unix socket for reporting flow data from each node.", + "DescriptionHTML": "

Configures local unix socket for reporting flow data from each node.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsMaxOriginalIPsIncluded", + "NameEnvVar": "FELIX_FlowLogsMaxOriginalIPsIncluded", + "NameYAML": "flowLogsMaxOriginalIPsIncluded", + "NameGoAPI": "FlowLogsMaxOriginalIPsIncluded", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "50", + "ParsedDefault": "50", + "ParsedDefaultJSON": "50", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "50", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies the number of unique IP addresses (if relevant) that should be included in Flow logs.", + "DescriptionHTML": "

Specifies the number of unique IP addresses (if relevant) that should be included in Flow logs.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsPolicyEvaluationMode", + "NameEnvVar": "FELIX_FlowLogsPolicyEvaluationMode", + "NameYAML": "flowLogsPolicyEvaluationMode", + "NameGoAPI": "FlowLogsPolicyEvaluationMode", + "StringSchema": "One of: `Continuous`, `OnNewConnection`", + "StringSchemaHTML": "One of: Continuous, OnNewConnection", + "StringDefault": "Continuous", + "ParsedDefault": "Continuous", + "ParsedDefaultJSON": "\"Continuous\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "Continuous", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Defines how policies are evaluated and reflected in flow logs.\nOnNewConnection - In this mode, staged policies are only evaluated when new connections are\nmade in the dataplane. Staged/active policy changes will not be reflected in the\n`pending_policies` field of flow logs for long lived connections.\nContinuous - Felix evaluates active flows on a regular basis to determine the rule\ntraces in the flow logs. Any policy updates that impact a flow will be reflected in the\npending_policies field, offering a near-real-time view of policy changes across flows.", + "DescriptionHTML": "

Defines how policies are evaluated and reflected in flow logs.\nOnNewConnection - In this mode, staged policies are only evaluated when new connections are\nmade in the dataplane. Staged/active policy changes will not be reflected in the\npending_policies field of flow logs for long lived connections.\nContinuous - Felix evaluates active flows on a regular basis to determine the rule\ntraces in the flow logs. Any policy updates that impact a flow will be reflected in the\npending_policies field, offering a near-real-time view of policy changes across flows.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsPolicyScope", + "NameEnvVar": "FELIX_FlowLogsPolicyScope", + "NameYAML": "flowLogsPolicyScope", + "NameGoAPI": "FlowLogsPolicyScope", + "StringSchema": "One of: `AllPolicies`, `EndpointPolicies`", + "StringSchemaHTML": "One of: AllPolicies, EndpointPolicies", + "StringDefault": "EndpointPolicies", + "ParsedDefault": "EndpointPolicies", + "ParsedDefaultJSON": "\"EndpointPolicies\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "EndpointPolicies", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls which policies are included in flow logs.\nAllPolicies - Processes both transit policies for the local node and\nendpoint policies derived from packet source/destination IPs. Provides comprehensive\nvisibility into all policy evaluations but increases log volume.\nEndpointPolicies - Processes only policies for endpoints identified as the source\nor destination of the packet (whether workload or host endpoints).", + "DescriptionHTML": "

Controls which policies are included in flow logs.\nAllPolicies - Processes both transit policies for the local node and\nendpoint policies derived from packet source/destination IPs. Provides comprehensive\nvisibility into all policy evaluations but increases log volume.\nEndpointPolicies - Processes only policies for endpoints identified as the source\nor destination of the packet (whether workload or host endpoints).

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Flow logs: file reports", + "GroupWithSortPrefix": "40 Flow logs: file reports", + "NameConfigFile": "FlowLogsPositionFilePath", + "NameEnvVar": "FELIX_FlowLogsPositionFilePath", + "NameYAML": "flowLogsPositionFilePath", + "NameGoAPI": "FlowLogsPositionFilePath", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "/var/log/calico/flows.log.pos", + "ParsedDefault": "/var/log/calico/flows.log.pos", + "ParsedDefaultJSON": "\"/var/log/calico/flows.log.pos\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "/var/log/calico/flows.log.pos", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used specify the position of the external pipeline that reads flow logs. Default is /var/log/calico/flows.log.pos.\nThis parameter only takes effect when FlowLogsDynamicAggregationEnabled is set to true.", + "DescriptionHTML": "

Used specify the position of the external pipeline that reads flow logs. Default is /var/log/calico/flows.log.pos.\nThis parameter only takes effect when FlowLogsDynamicAggregationEnabled is set to true.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "DNS logs / policy", + "Fields": [ + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSCacheEpoch", + "NameEnvVar": "FELIX_DNSCacheEpoch", + "NameYAML": "dnsCacheEpoch", + "NameGoAPI": "DNSCacheEpoch", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "0", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "An arbitrary number that can be changed, at runtime, to tell Felix to discard all its\nlearnt DNS information. .", + "DescriptionHTML": "

An arbitrary number that can be changed, at runtime, to tell Felix to discard all its\nlearnt DNS information. .

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSCacheFile", + "NameEnvVar": "FELIX_DNSCacheFile", + "NameYAML": "dnsCacheFile", + "NameGoAPI": "DNSCacheFile", + "StringSchema": "Path to file", + "StringSchemaHTML": "Path to file", + "StringDefault": "/var/run/calico/felix-dns-cache.txt", + "ParsedDefault": "/var/run/calico/felix-dns-cache.txt", + "ParsedDefaultJSON": "\"/var/run/calico/felix-dns-cache.txt\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "/var/run/calico/felix-dns-cache.txt", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The name of the file that Felix uses to preserve learnt DNS information when restarting. .", + "DescriptionHTML": "

The name of the file that Felix uses to preserve learnt DNS information when restarting. .

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSCacheSaveInterval", + "NameEnvVar": "FELIX_DNSCacheSaveInterval", + "NameYAML": "dnsCacheSaveInterval", + "NameGoAPI": "DNSCacheSaveInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "60", + "ParsedDefault": "1m0s", + "ParsedDefaultJSON": "60000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "1m0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The periodic interval at which Felix saves learnt DNS information to the cache file. .", + "DescriptionHTML": "

The periodic interval at which Felix saves learnt DNS information to the cache file. .

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSExtraTTL", + "NameEnvVar": "FELIX_DNSExtraTTL", + "NameYAML": "dnsExtraTTL", + "NameGoAPI": "DNSExtraTTL", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "0", + "ParsedDefault": "0s", + "ParsedDefaultJSON": "0", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Extra time to keep IPs and alias names that are learnt from DNS, in addition to each name\nor IP's advertised TTL. .", + "DescriptionHTML": "

Extra time to keep IPs and alias names that are learnt from DNS, in addition to each name\nor IP's advertised TTL. .

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSLogsFileAggregationKind", + "NameEnvVar": "FELIX_DNSLogsFileAggregationKind", + "NameYAML": "dnsLogsFileAggregationKind", + "NameGoAPI": "DNSLogsFileAggregationKind", + "StringSchema": "Integer: [0,1]", + "StringSchemaHTML": "Integer: [0,1]", + "StringDefault": "1", + "ParsedDefault": "1", + "ParsedDefaultJSON": "1", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "One of: `0`, `1`.", + "YAMLEnumValues": [ + "0", + "1" + ], + "YAMLSchemaHTML": "One of: 0, 1.", + "YAMLDefault": "1", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to choose the type of aggregation for DNS log entries.\n.\nAccepted values are 0 and 1.\n0 - No aggregation.\n1 - Aggregate over clients with the same name prefix.", + "DescriptionHTML": "

Used to choose the type of aggregation for DNS log entries.\n.\nAccepted values are 0 and 1.\n0 - No aggregation.\n1 - Aggregate over clients with the same name prefix.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSLogsFileDirectory", + "NameEnvVar": "FELIX_DNSLogsFileDirectory", + "NameYAML": "dnsLogsFileDirectory", + "NameGoAPI": "DNSLogsFileDirectory", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "/var/log/calico/dnslogs", + "ParsedDefault": "/var/log/calico/dnslogs", + "ParsedDefaultJSON": "\"/var/log/calico/dnslogs\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "/var/log/calico/dnslogs", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the directory where DNS log files are stored.", + "DescriptionHTML": "

Sets the directory where DNS log files are stored.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSLogsFileEnabled", + "NameEnvVar": "FELIX_DNSLogsFileEnabled", + "NameYAML": "dnsLogsFileEnabled", + "NameGoAPI": "DNSLogsFileEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls logging DNS logs to a file. If false no DNS logging to file will occur.", + "DescriptionHTML": "

Controls logging DNS logs to a file. If false no DNS logging to file will occur.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSLogsFileIncludeLabels", + "NameEnvVar": "FELIX_DNSLogsFileIncludeLabels", + "NameYAML": "dnsLogsFileIncludeLabels", + "NameGoAPI": "DNSLogsFileIncludeLabels", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to configure if endpoint labels are included in a DNS log entry written to file.", + "DescriptionHTML": "

Used to configure if endpoint labels are included in a DNS log entry written to file.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSLogsFileMaxFileSizeMB", + "NameEnvVar": "FELIX_DNSLogsFileMaxFileSizeMB", + "NameYAML": "dnsLogsFileMaxFileSizeMB", + "NameGoAPI": "DNSLogsFileMaxFileSizeMB", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "100", + "ParsedDefault": "100", + "ParsedDefaultJSON": "100", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "100", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the max size in MB of DNS log files before rotation.", + "DescriptionHTML": "

Sets the max size in MB of DNS log files before rotation.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSLogsFileMaxFiles", + "NameEnvVar": "FELIX_DNSLogsFileMaxFiles", + "NameYAML": "dnsLogsFileMaxFiles", + "NameGoAPI": "DNSLogsFileMaxFiles", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "5", + "ParsedDefault": "5", + "ParsedDefaultJSON": "5", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "5", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the number of DNS log files to keep.", + "DescriptionHTML": "

Sets the number of DNS log files to keep.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSLogsFilePerNodeLimit", + "NameEnvVar": "FELIX_DNSLogsFilePerNodeLimit", + "NameYAML": "dnsLogsFilePerNodeLimit", + "NameGoAPI": "DNSLogsFilePerNodeLimit", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "0", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "0", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Limit on the number of DNS logs that can be emitted within each flush interval. When\nthis limit has been reached, Felix counts the number of unloggable DNS responses within\nthe flush interval, and emits a WARNING log with that count at the same time as it\nflushes the buffered DNS logs.", + "DescriptionHTML": "

Limit on the number of DNS logs that can be emitted within each flush interval. When\nthis limit has been reached, Felix counts the number of unloggable DNS responses within\nthe flush interval, and emits a WARNING log with that count at the same time as it\nflushes the buffered DNS logs.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSLogsFlushInterval", + "NameEnvVar": "FELIX_DNSLogsFlushInterval", + "NameYAML": "dnsLogsFlushInterval", + "NameGoAPI": "DNSLogsFlushInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "300", + "ParsedDefault": "5m0s", + "ParsedDefaultJSON": "300000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "5m0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures the interval at which Felix exports DNS logs.", + "DescriptionHTML": "

Configures the interval at which Felix exports DNS logs.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSLogsLatency", + "NameEnvVar": "FELIX_DNSLogsLatency", + "NameYAML": "dnsLogsLatency", + "NameGoAPI": "DNSLogsLatency", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Indicates to include measurements of DNS request/response latency in each DNS log.", + "DescriptionHTML": "

Indicates to include measurements of DNS request/response latency in each DNS log.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSPacketsNfqueueID", + "NameEnvVar": "FELIX_DNSPacketsNfqueueID", + "NameYAML": "dnsPacketsNfqueueID", + "NameGoAPI": "DNSPacketsNfqueueID", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "101", + "ParsedDefault": "101", + "ParsedDefaultJSON": "101", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "101", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The NFQUEUE ID to use for capturing DNS packets to ensure programming IPSets occurs before\nthe response is released. Used when DNSPolicyMode is DelayDNSResponse.", + "DescriptionHTML": "

The NFQUEUE ID to use for capturing DNS packets to ensure programming IPSets occurs before\nthe response is released. Used when DNSPolicyMode is DelayDNSResponse.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSPacketsNfqueueMaxHoldDuration", + "NameEnvVar": "FELIX_DNSPacketsNfqueueMaxHoldDuration", + "NameYAML": "dnsPacketsNfqueueMaxHoldDuration", + "NameGoAPI": "DNSPacketsNfqueueMaxHoldDuration", + "StringSchema": "Milliseconds (floating point)", + "StringSchemaHTML": "Milliseconds (floating point)", + "StringDefault": "3000", + "ParsedDefault": "3s", + "ParsedDefaultJSON": "3000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "3s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The max length of time to hold on to a DNS response while waiting for the\nthe dataplane to be programmed. Used when DNSPolicyMode is DelayDNSResponse.", + "DescriptionHTML": "

The max length of time to hold on to a DNS response while waiting for the\nthe dataplane to be programmed. Used when DNSPolicyMode is DelayDNSResponse.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSPacketsNfqueueSize", + "NameEnvVar": "FELIX_DNSPacketsNfqueueSize", + "NameYAML": "dnsPacketsNfqueueSize", + "NameGoAPI": "DNSPacketsNfqueueSize", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "100", + "ParsedDefault": "100", + "ParsedDefaultJSON": "100", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "100", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The size of the NFQUEUE for captured DNS packets. This is the maximum number of DNS\npackets that may be queued awaiting programming in the dataplane. Used when DNSPolicyMode is DelayDNSResponse.", + "DescriptionHTML": "

The size of the NFQUEUE for captured DNS packets. This is the maximum number of DNS\npackets that may be queued awaiting programming in the dataplane. Used when DNSPolicyMode is DelayDNSResponse.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSPolicyMode", + "NameEnvVar": "FELIX_DNSPolicyMode", + "NameYAML": "dnsPolicyMode", + "NameGoAPI": "DNSPolicyMode", + "StringSchema": "One of: `DelayDNSResponse`, `DelayDeniedPacket`, `Inline`, `NoDelay`", + "StringSchemaHTML": "One of: DelayDNSResponse, DelayDeniedPacket, Inline, NoDelay", + "StringDefault": "DelayDeniedPacket", + "ParsedDefault": "DelayDeniedPacket", + "ParsedDefaultJSON": "\"DelayDeniedPacket\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"DelayDNSResponse\"`, `\"DelayDeniedPacket\"`, `\"Inline\"`, `\"NoDelay\"`.", + "YAMLEnumValues": [ + "DelayDNSResponse", + "DelayDeniedPacket", + "Inline", + "NoDelay" + ], + "YAMLSchemaHTML": "One of: \"DelayDNSResponse\", \"DelayDeniedPacket\", \"Inline\", \"NoDelay\".", + "YAMLDefault": "DelayDeniedPacket", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Specifies how DNS policy programming will be handled.\nDelayDeniedPacket - Felix delays any denied packet that traversed a policy that included egress domain matches,\nbut did not match. The packet is released after a fixed time, or after the destination IP address was programmed.\nDelayDNSResponse - Felix delays any DNS response until related IPSets are programmed. This introduces some\nlatency to all DNS packets (even when no IPSet programming is required), but it ensures policy hit statistics\nare accurate. This is the recommended setting when you are making use of staged policies or policy rule hit\nstatistics.\nNoDelay - Felix does not introduce any delay to the packets. DNS rules may not have been programmed by the time\nthe first packet traverses the policy rules. Client applications need to handle reconnection attempts if initial\nconnection attempts fail. This may be problematic for some applications or for very low DNS TTLs.\n\nInline - Parses DNS response inline with DNS response packet processing within IPTables.\nThis guarantees the DNS rules reflect any change immediately.\nThis mode works for iptables only and matches the same mode for BPFDNSPolicyMode.\nThis setting is ignored on Windows and \"NoDelay\" is always used.\n\nThis setting is ignored by eBPF and BPFDNSPolicyMode is used instead.\n\nThis field has no effect in NFTables mode. Please use NFTablesDNSPolicyMode instead.", + "DescriptionHTML": "

Specifies how DNS policy programming will be handled.\nDelayDeniedPacket - Felix delays any denied packet that traversed a policy that included egress domain matches,\nbut did not match. The packet is released after a fixed time, or after the destination IP address was programmed.\nDelayDNSResponse - Felix delays any DNS response until related IPSets are programmed. This introduces some\nlatency to all DNS packets (even when no IPSet programming is required), but it ensures policy hit statistics\nare accurate. This is the recommended setting when you are making use of staged policies or policy rule hit\nstatistics.\nNoDelay - Felix does not introduce any delay to the packets. DNS rules may not have been programmed by the time\nthe first packet traverses the policy rules. Client applications need to handle reconnection attempts if initial\nconnection attempts fail. This may be problematic for some applications or for very low DNS TTLs.

\n

Inline - Parses DNS response inline with DNS response packet processing within IPTables.\nThis guarantees the DNS rules reflect any change immediately.\nThis mode works for iptables only and matches the same mode for BPFDNSPolicyMode.\nThis setting is ignored on Windows and \"NoDelay\" is always used.

\n

This setting is ignored by eBPF and BPFDNSPolicyMode is used instead.

\n

This field has no effect in NFTables mode. Please use NFTablesDNSPolicyMode instead.

", + "UserEditable": true, + "GoType": "*v3.DNSPolicyMode", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSPolicyNfqueueID", + "NameEnvVar": "FELIX_DNSPolicyNfqueueID", + "NameYAML": "dnsPolicyNfqueueID", + "NameGoAPI": "DNSPolicyNfqueueID", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "100", + "ParsedDefault": "100", + "ParsedDefaultJSON": "100", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "100", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The NFQUEUE ID to use for DNS Policy re-evaluation when the domains IP hasn't been programmed\nto ipsets yet. Used when DNSPolicyMode is DelayDeniedPacket.", + "DescriptionHTML": "

The NFQUEUE ID to use for DNS Policy re-evaluation when the domains IP hasn't been programmed\nto ipsets yet. Used when DNSPolicyMode is DelayDeniedPacket.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSPolicyNfqueueSize", + "NameEnvVar": "FELIX_DNSPolicyNfqueueSize", + "NameYAML": "dnsPolicyNfqueueSize", + "NameGoAPI": "DNSPolicyNfqueueSize", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "255", + "ParsedDefault": "255", + "ParsedDefaultJSON": "255", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "255", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "DNSPolicyNfqueueID is the size of the NFQUEUE for DNS policy re-evaluation. This is the maximum number of denied\npackets that may be queued up pending re-evaluation.\nUsed when DNSPolicyMode is DelayDeniedPacket.", + "DescriptionHTML": "

DNSPolicyNfqueueID is the size of the NFQUEUE for DNS policy re-evaluation. This is the maximum number of denied\npackets that may be queued up pending re-evaluation.\nUsed when DNSPolicyMode is DelayDeniedPacket.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "DNS logs / policy", + "GroupWithSortPrefix": "50 DNS logs / policy", + "NameConfigFile": "DNSTrustedServers", + "NameEnvVar": "FELIX_DNSTrustedServers", + "NameYAML": "dnsTrustedServers", + "NameGoAPI": "DNSTrustedServers", + "StringSchema": "Comma-delimited list of DNS servers. Each entry can be: ``, an `:` (IPv6 addresses must be wrapped in square brackets), or, a Kubernetes service name `k8s-service:(namespace/)service-name`.", + "StringSchemaHTML": "Comma-delimited list of DNS servers. Each entry can be: <IP address>, an <IP address>:<port> (IPv6 addresses must be wrapped in square brackets), or, a Kubernetes service name k8s-service:(namespace/)service-name.", + "StringDefault": "k8s-service:kube-dns", + "ParsedDefault": "[]", + "ParsedDefaultJSON": "[]", + "ParsedType": "[]config.ServerPort", + "YAMLType": "array", + "YAMLSchema": "List of strings: `[\"\", ...]`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "List of strings: [\"<string>\", ...].", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The DNS servers that Felix should trust. Each entry here must be `[:]` - indicating an\nexplicit DNS server IP - or `k8s-service:[/][:port]` - indicating a Kubernetes DNS\nservice. `` defaults to the first service port, or 53 for an IP, and `` to\n`kube-system`. An IPv6 address with a port must use the square brackets convention, for example\n`[fd00:83a6::12]:5353`.Note that Felix (calico-node) will need RBAC permission to read the details of\neach service specified by a `k8s-service:...` form. .", + "DescriptionHTML": "

The DNS servers that Felix should trust. Each entry here must be <ip>[:<port>] - indicating an\nexplicit DNS server IP - or k8s-service:[<namespace>/]<name>[:port] - indicating a Kubernetes DNS\nservice. <port> defaults to the first service port, or 53 for an IP, and <namespace> to\nkube-system. An IPv6 address with a port must use the square brackets convention, for example\n[fd00:83a6::12]:5353.Note that Felix (calico-node) will need RBAC permission to read the details of\neach service specified by a k8s-service:... form. .

", + "UserEditable": true, + "GoType": "*[]string", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "L7 logs", + "Fields": [ + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFileAggregationDestinationInfo", + "NameEnvVar": "FELIX_L7LogsFileAggregationDestinationInfo", + "NameYAML": "l7LogsFileAggregationDestinationInfo", + "NameGoAPI": "L7LogsFileAggregationDestinationInfo", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "IncludeL7DestinationInfo", + "ParsedDefault": "IncludeL7DestinationInfo", + "ParsedDefaultJSON": "\"IncludeL7DestinationInfo\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `ExcludeL7DestinationInfo`, `IncludeL7DestinationInfo`.", + "YAMLEnumValues": [ + "ExcludeL7DestinationInfo", + "IncludeL7DestinationInfo" + ], + "YAMLSchemaHTML": "One of: ExcludeL7DestinationInfo, IncludeL7DestinationInfo.", + "YAMLDefault": "IncludeL7DestinationInfo", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to choose the type of aggregation for the destination metadata on L7 log entries.\n.\nAccepted values are IncludeL7DestinationInfo and ExcludeL7DestinationInfo.\nIncludeL7DestinationInfo - Include destination metadata in the logs.\nExcludeL7DestinationInfo - Aggregate over all other fields ignoring the destination aggregated name, namespace, and type.", + "DescriptionHTML": "

Used to choose the type of aggregation for the destination metadata on L7 log entries.\n.\nAccepted values are IncludeL7DestinationInfo and ExcludeL7DestinationInfo.\nIncludeL7DestinationInfo - Include destination metadata in the logs.\nExcludeL7DestinationInfo - Aggregate over all other fields ignoring the destination aggregated name, namespace, and type.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFileAggregationHTTPHeaderInfo", + "NameEnvVar": "FELIX_L7LogsFileAggregationHTTPHeaderInfo", + "NameYAML": "l7LogsFileAggregationHTTPHeaderInfo", + "NameGoAPI": "L7LogsFileAggregationHTTPHeaderInfo", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "ExcludeL7HTTPHeaderInfo", + "ParsedDefault": "ExcludeL7HTTPHeaderInfo", + "ParsedDefaultJSON": "\"ExcludeL7HTTPHeaderInfo\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `ExcludeL7HTTPHeaderInfo`, `IncludeL7HTTPHeaderInfo`.", + "YAMLEnumValues": [ + "ExcludeL7HTTPHeaderInfo", + "IncludeL7HTTPHeaderInfo" + ], + "YAMLSchemaHTML": "One of: ExcludeL7HTTPHeaderInfo, IncludeL7HTTPHeaderInfo.", + "YAMLDefault": "ExcludeL7HTTPHeaderInfo", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to choose the type of aggregation for HTTP header data on L7 log entries.\n.\nAccepted values are IncludeL7HTTPHeaderInfo and ExcludeL7HTTPHeaderInfo.\nIncludeL7HTTPHeaderInfo - Include HTTP header data in the logs.\nExcludeL7HTTPHeaderInfo - Aggregate over all other fields ignoring the user agent and log type.", + "DescriptionHTML": "

Used to choose the type of aggregation for HTTP header data on L7 log entries.\n.\nAccepted values are IncludeL7HTTPHeaderInfo and ExcludeL7HTTPHeaderInfo.\nIncludeL7HTTPHeaderInfo - Include HTTP header data in the logs.\nExcludeL7HTTPHeaderInfo - Aggregate over all other fields ignoring the user agent and log type.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFileAggregationHTTPMethod", + "NameEnvVar": "FELIX_L7LogsFileAggregationHTTPMethod", + "NameYAML": "l7LogsFileAggregationHTTPMethod", + "NameGoAPI": "L7LogsFileAggregationHTTPMethod", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "IncludeL7HTTPMethod", + "ParsedDefault": "IncludeL7HTTPMethod", + "ParsedDefaultJSON": "\"IncludeL7HTTPMethod\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `ExcludeL7HTTPMethod`, `IncludeL7HTTPMethod`.", + "YAMLEnumValues": [ + "ExcludeL7HTTPMethod", + "IncludeL7HTTPMethod" + ], + "YAMLSchemaHTML": "One of: ExcludeL7HTTPMethod, IncludeL7HTTPMethod.", + "YAMLDefault": "IncludeL7HTTPMethod", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to choose the type of aggregation for the HTTP request method on L7 log entries.\n.\nAccepted values are IncludeL7HTTPMethod and ExcludeL7HTTPMethod.\nIncludeL7HTTPMethod - Include HTTP method in the logs.\nExcludeL7HTTPMethod - Aggregate over all other fields ignoring the HTTP method.", + "DescriptionHTML": "

Used to choose the type of aggregation for the HTTP request method on L7 log entries.\n.\nAccepted values are IncludeL7HTTPMethod and ExcludeL7HTTPMethod.\nIncludeL7HTTPMethod - Include HTTP method in the logs.\nExcludeL7HTTPMethod - Aggregate over all other fields ignoring the HTTP method.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFileAggregationNumURLPath", + "NameEnvVar": "FELIX_L7LogsFileAggregationNumURLPath", + "NameYAML": "l7LogsFileAggregationNumURLPath", + "NameGoAPI": "L7LogsFileAggregationNumURLPath", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "5", + "ParsedDefault": "5", + "ParsedDefaultJSON": "5", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "5", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to choose the number of components in the url path to display.\nThis allows for the url to be truncated in case parts of the path provide no value. Setting this value\nto negative will allow all parts of the path to be displayed.\n.", + "DescriptionHTML": "

Used to choose the number of components in the url path to display.\nThis allows for the url to be truncated in case parts of the path provide no value. Setting this value\nto negative will allow all parts of the path to be displayed.\n.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFileAggregationResponseCode", + "NameEnvVar": "FELIX_L7LogsFileAggregationResponseCode", + "NameYAML": "l7LogsFileAggregationResponseCode", + "NameGoAPI": "L7LogsFileAggregationResponseCode", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "IncludeL7ResponseCode", + "ParsedDefault": "IncludeL7ResponseCode", + "ParsedDefaultJSON": "\"IncludeL7ResponseCode\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `ExcludeL7ResponseCode`, `IncludeL7ResponseCode`.", + "YAMLEnumValues": [ + "ExcludeL7ResponseCode", + "IncludeL7ResponseCode" + ], + "YAMLSchemaHTML": "One of: ExcludeL7ResponseCode, IncludeL7ResponseCode.", + "YAMLDefault": "IncludeL7ResponseCode", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to choose the type of aggregation for the response code on L7 log entries.\n.\nAccepted values are IncludeL7ResponseCode and ExcludeL7ResponseCode.\nIncludeL7ResponseCode - Include the response code in the logs.\nExcludeL7ResponseCode - Aggregate over all other fields ignoring the response code.", + "DescriptionHTML": "

Used to choose the type of aggregation for the response code on L7 log entries.\n.\nAccepted values are IncludeL7ResponseCode and ExcludeL7ResponseCode.\nIncludeL7ResponseCode - Include the response code in the logs.\nExcludeL7ResponseCode - Aggregate over all other fields ignoring the response code.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFileAggregationServiceInfo", + "NameEnvVar": "FELIX_L7LogsFileAggregationServiceInfo", + "NameYAML": "l7LogsFileAggregationServiceInfo", + "NameGoAPI": "L7LogsFileAggregationServiceInfo", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "IncludeL7ServiceInfo", + "ParsedDefault": "IncludeL7ServiceInfo", + "ParsedDefaultJSON": "\"IncludeL7ServiceInfo\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `ExcludeL7ServiceInfo`, `IncludeL7ServiceInfo`.", + "YAMLEnumValues": [ + "ExcludeL7ServiceInfo", + "IncludeL7ServiceInfo" + ], + "YAMLSchemaHTML": "One of: ExcludeL7ServiceInfo, IncludeL7ServiceInfo.", + "YAMLDefault": "IncludeL7ServiceInfo", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to choose the type of aggregation for the service data on L7 log entries.\n.\nAccepted values are IncludeL7ServiceInfo and ExcludeL7ServiceInfo.\nIncludeL7ServiceInfo - Include service data in the logs.\nExcludeL7ServiceInfo - Aggregate over all other fields ignoring the service name, namespace, and port.", + "DescriptionHTML": "

Used to choose the type of aggregation for the service data on L7 log entries.\n.\nAccepted values are IncludeL7ServiceInfo and ExcludeL7ServiceInfo.\nIncludeL7ServiceInfo - Include service data in the logs.\nExcludeL7ServiceInfo - Aggregate over all other fields ignoring the service name, namespace, and port.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFileAggregationSourceInfo", + "NameEnvVar": "FELIX_L7LogsFileAggregationSourceInfo", + "NameYAML": "l7LogsFileAggregationSourceInfo", + "NameGoAPI": "L7LogsFileAggregationSourceInfo", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "IncludeL7SourceInfoNoPort", + "ParsedDefault": "IncludeL7SourceInfoNoPort", + "ParsedDefaultJSON": "\"IncludeL7SourceInfoNoPort\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `ExcludeL7SourceInfo`, `IncludeL7SourceInfo`, `IncludeL7SourceInfoNoPort`.", + "YAMLEnumValues": [ + "ExcludeL7SourceInfo", + "IncludeL7SourceInfo", + "IncludeL7SourceInfoNoPort" + ], + "YAMLSchemaHTML": "One of: ExcludeL7SourceInfo, IncludeL7SourceInfo, IncludeL7SourceInfoNoPort.", + "YAMLDefault": "IncludeL7SourceInfoNoPort", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "L7LogsFileAggregationExcludeSourceInfo is used to choose the type of aggregation for the source metadata on L7 log entries.\n.\nAccepted values are IncludeL7SourceInfo, IncludeL7SourceInfoNoPort, and ExcludeL7SourceInfo.\nIncludeL7SourceInfo - Include source metadata in the logs.\nIncludeL7SourceInfoNoPort - Include source metadata in the logs excluding the source port.\nExcludeL7SourceInfo - Aggregate over all other fields ignoring the source aggregated name, namespace, and type.", + "DescriptionHTML": "

L7LogsFileAggregationExcludeSourceInfo is used to choose the type of aggregation for the source metadata on L7 log entries.\n.\nAccepted values are IncludeL7SourceInfo, IncludeL7SourceInfoNoPort, and ExcludeL7SourceInfo.\nIncludeL7SourceInfo - Include source metadata in the logs.\nIncludeL7SourceInfoNoPort - Include source metadata in the logs excluding the source port.\nExcludeL7SourceInfo - Aggregate over all other fields ignoring the source aggregated name, namespace, and type.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFileAggregationTLSSNI", + "NameEnvVar": "FELIX_L7LogsFileAggregationTLSSNI", + "NameYAML": "l7LogsFileAggregationTLSSNI", + "NameGoAPI": "L7LogsFileAggregationTLSSNI", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "IncludeL7TLSSNI", + "ParsedDefault": "IncludeL7TLSSNI", + "ParsedDefaultJSON": "\"IncludeL7TLSSNI\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `ExcludeL7TLSSNI`, `IncludeL7TLSSNI`.", + "YAMLEnumValues": [ + "ExcludeL7TLSSNI", + "IncludeL7TLSSNI" + ], + "YAMLSchemaHTML": "One of: ExcludeL7TLSSNI, IncludeL7TLSSNI.", + "YAMLDefault": "IncludeL7TLSSNI", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether the TLS Server Name Indication (SNI)\nparticipates in the aggregation key for L7 logs.\n\nAccepted values:\nIncludeL7TLSSNI - Include the SNI in the aggregation key.\nExcludeL7TLSSNI - Aggregate over all other fields ignoring the SNI entirely.", + "DescriptionHTML": "

Controls whether the TLS Server Name Indication (SNI)\nparticipates in the aggregation key for L7 logs.

\n

Accepted values:\nIncludeL7TLSSNI - Include the SNI in the aggregation key.\nExcludeL7TLSSNI - Aggregate over all other fields ignoring the SNI entirely.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFileAggregationTrimURL", + "NameEnvVar": "FELIX_L7LogsFileAggregationTrimURL", + "NameYAML": "l7LogsFileAggregationTrimURL", + "NameGoAPI": "L7LogsFileAggregationTrimURL", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "IncludeL7FullURL", + "ParsedDefault": "IncludeL7FullURL", + "ParsedDefaultJSON": "\"IncludeL7FullURL\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `ExcludeL7URL`, `IncludeL7FullURL`, `TrimURLQuery`, `TrimURLQueryAndPath`.", + "YAMLEnumValues": [ + "ExcludeL7URL", + "IncludeL7FullURL", + "TrimURLQuery", + "TrimURLQueryAndPath" + ], + "YAMLSchemaHTML": "One of: ExcludeL7URL, IncludeL7FullURL, TrimURLQuery, TrimURLQueryAndPath.", + "YAMLDefault": "IncludeL7FullURL", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to choose the type of aggregation for the url on L7 log entries.\n.\nAccepted values:\nIncludeL7FullURL - Include the full URL up to however many path components are allowed by L7LogsFileAggregationNumURLPath.\nTrimURLQuery - Aggregate over all other fields ignoring the query parameters on the URL.\nTrimURLQueryAndPath - Aggregate over all other fields and the base URL only.\nExcludeL7URL - Aggregate over all other fields ignoring the URL entirely.", + "DescriptionHTML": "

Used to choose the type of aggregation for the url on L7 log entries.\n.\nAccepted values:\nIncludeL7FullURL - Include the full URL up to however many path components are allowed by L7LogsFileAggregationNumURLPath.\nTrimURLQuery - Aggregate over all other fields ignoring the query parameters on the URL.\nTrimURLQueryAndPath - Aggregate over all other fields and the base URL only.\nExcludeL7URL - Aggregate over all other fields ignoring the URL entirely.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFileAggregationURLCharLimit", + "NameEnvVar": "FELIX_L7LogsFileAggregationURLCharLimit", + "NameYAML": "l7LogsFileAggregationURLCharLimit", + "NameGoAPI": "L7LogsFileAggregationURLCharLimit", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "250", + "ParsedDefault": "250", + "ParsedDefaultJSON": "250", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "250", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Limit on the length of the URL collected in L7 logs. When a URL length reaches this limit\nit is sliced off, and the sliced URL is sent to log storage.", + "DescriptionHTML": "

Limit on the length of the URL collected in L7 logs. When a URL length reaches this limit\nit is sliced off, and the sliced URL is sent to log storage.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFileDirectory", + "NameEnvVar": "FELIX_L7LogsFileDirectory", + "NameYAML": "l7LogsFileDirectory", + "NameGoAPI": "L7LogsFileDirectory", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "/var/log/calico/l7logs", + "ParsedDefault": "/var/log/calico/l7logs", + "ParsedDefaultJSON": "\"/var/log/calico/l7logs\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "/var/log/calico/l7logs", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the directory where L7 log files are stored.", + "DescriptionHTML": "

Sets the directory where L7 log files are stored.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFileEnabled", + "NameEnvVar": "FELIX_L7LogsFileEnabled", + "NameYAML": "l7LogsFileEnabled", + "NameGoAPI": "L7LogsFileEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls logging L7 logs to a file. If false no L7 logging to file will occur.", + "DescriptionHTML": "

Controls logging L7 logs to a file. If false no L7 logging to file will occur.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFileMaxFileSizeMB", + "NameEnvVar": "FELIX_L7LogsFileMaxFileSizeMB", + "NameYAML": "l7LogsFileMaxFileSizeMB", + "NameGoAPI": "L7LogsFileMaxFileSizeMB", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "100", + "ParsedDefault": "100", + "ParsedDefaultJSON": "100", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "100", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the max size in MB of L7 log files before rotation.", + "DescriptionHTML": "

Sets the max size in MB of L7 log files before rotation.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFileMaxFiles", + "NameEnvVar": "FELIX_L7LogsFileMaxFiles", + "NameYAML": "l7LogsFileMaxFiles", + "NameGoAPI": "L7LogsFileMaxFiles", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "5", + "ParsedDefault": "5", + "ParsedDefaultJSON": "5", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "5", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets the number of L7 log files to keep.", + "DescriptionHTML": "

Sets the number of L7 log files to keep.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFilePerNodeLimit", + "NameEnvVar": "FELIX_L7LogsFilePerNodeLimit", + "NameYAML": "l7LogsFilePerNodeLimit", + "NameGoAPI": "L7LogsFilePerNodeLimit", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "1500", + "ParsedDefault": "1500", + "ParsedDefaultJSON": "1500", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "1500", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Limit on the number of L7 logs that can be emitted within each flush interval. When\nthis limit has been reached, Felix counts the number of unloggable L7 responses within\nthe flush interval, and emits a WARNING log with that count at the same time as it\nflushes the buffered L7 logs. A value of 0 means no limit.", + "DescriptionHTML": "

Limit on the number of L7 logs that can be emitted within each flush interval. When\nthis limit has been reached, Felix counts the number of unloggable L7 responses within\nthe flush interval, and emits a WARNING log with that count at the same time as it\nflushes the buffered L7 logs. A value of 0 means no limit.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "L7 logs", + "GroupWithSortPrefix": "50 L7 logs", + "NameConfigFile": "L7LogsFlushInterval", + "NameEnvVar": "FELIX_L7LogsFlushInterval", + "NameYAML": "l7LogsFlushInterval", + "NameGoAPI": "L7LogsFlushInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "300", + "ParsedDefault": "5m0s", + "ParsedDefaultJSON": "300000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "5m0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Configures the interval at which Felix exports L7 logs.", + "DescriptionHTML": "

Configures the interval at which Felix exports L7 logs.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "AWS integration", + "Fields": [ + { + "Group": "AWS integration", + "GroupWithSortPrefix": "60 AWS integration", + "NameConfigFile": "AWSRequestTimeout", + "NameEnvVar": "FELIX_AWSRequestTimeout", + "NameYAML": "awsRequestTimeout", + "NameGoAPI": "AWSRequestTimeout", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "30", + "ParsedDefault": "30s", + "ParsedDefaultJSON": "30000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "30s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The timeout on AWS API requests.", + "DescriptionHTML": "

The timeout on AWS API requests.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "AWS integration", + "GroupWithSortPrefix": "60 AWS integration", + "NameConfigFile": "AWSSecondaryIPRoutingRulePriority", + "NameEnvVar": "FELIX_AWSSecondaryIPRoutingRulePriority", + "NameYAML": "awsSecondaryIPRoutingRulePriority", + "NameGoAPI": "AWSSecondaryIPRoutingRulePriority", + "StringSchema": "Integer: [0,4294967295]", + "StringSchemaHTML": "Integer: [0,4294967295]", + "StringDefault": "101", + "ParsedDefault": "101", + "ParsedDefaultJSON": "101", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [0,4294967295]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [0,4294967295]", + "YAMLDefault": "101", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the priority that Felix will use for routing rules when programming\nthem for AWS Secondary IP support.", + "DescriptionHTML": "

Controls the priority that Felix will use for routing rules when programming\nthem for AWS Secondary IP support.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "AWS integration", + "GroupWithSortPrefix": "60 AWS integration", + "NameConfigFile": "AWSSecondaryIPSupport", + "NameEnvVar": "FELIX_AWSSecondaryIPSupport", + "NameYAML": "awsSecondaryIPSupport", + "NameGoAPI": "AWSSecondaryIPSupport", + "StringSchema": "One of: `Disabled`, `EnabledENIPerWorkload`, `Enabled`", + "StringSchemaHTML": "One of: Disabled, EnabledENIPerWorkload, Enabled", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `Enabled`, `EnabledENIPerWorkload`.", + "YAMLEnumValues": [ + "Disabled", + "Enabled", + "EnabledENIPerWorkload" + ], + "YAMLSchemaHTML": "One of: Disabled, Enabled, EnabledENIPerWorkload.", + "YAMLDefault": "Disabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix will try to provision AWS secondary ENIs for\nworkloads that have IPs from IP pools that are configured with an AWS subnet ID. If the field is set to\n\"EnabledENIPerWorkload\" then each workload with an AWS-backed IP will be assigned its own secondary ENI.\nIf set to \"Enabled\" then each workload with an AWS-backed IP pool will be allocated a secondary IP address\non a secondary ENI; this mode requires additional IP pools to be provisioned for the host to claim IPs for\nthe primary IP of the secondary ENIs. Accepted value must be one of \"Enabled\", \"EnabledENIPerWorkload\" or\n\"Disabled\".", + "DescriptionHTML": "

Controls whether Felix will try to provision AWS secondary ENIs for\nworkloads that have IPs from IP pools that are configured with an AWS subnet ID. If the field is set to\n\"EnabledENIPerWorkload\" then each workload with an AWS-backed IP will be assigned its own secondary ENI.\nIf set to \"Enabled\" then each workload with an AWS-backed IP pool will be allocated a secondary IP address\non a secondary ENI; this mode requires additional IP pools to be provisioned for the host to claim IPs for\nthe primary IP of the secondary ENIs. Accepted value must be one of \"Enabled\", \"EnabledENIPerWorkload\" or\n\"Disabled\".

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "AWS integration", + "GroupWithSortPrefix": "60 AWS integration", + "NameConfigFile": "AWSSrcDstCheck", + "NameEnvVar": "FELIX_AWSSrcDstCheck", + "NameYAML": "awsSrcDstCheck", + "NameGoAPI": "AWSSrcDstCheck", + "StringSchema": "One of: `Disable`, `DoNothing`, `Enable`", + "StringSchemaHTML": "One of: Disable, DoNothing, Enable", + "StringDefault": "DoNothing", + "ParsedDefault": "DoNothing", + "ParsedDefaultJSON": "\"DoNothing\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `\"Disable\"`, `\"DoNothing\"`, `\"Enable\"`.", + "YAMLEnumValues": [ + "Disable", + "DoNothing", + "Enable" + ], + "YAMLSchemaHTML": "One of: \"Disable\", \"DoNothing\", \"Enable\".", + "YAMLDefault": "DoNothing", + "Required": true, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls whether Felix will try to change the \"source/dest check\" setting on the EC2 instance\non which it is running. A value of \"Disable\" will try to disable the source/dest check. Disabling the check\nallows for sending workload traffic without encapsulation within the same AWS subnet.", + "DescriptionHTML": "

Controls whether Felix will try to change the \"source/dest check\" setting on the EC2 instance\non which it is running. A value of \"Disable\" will try to disable the source/dest check. Disabling the check\nallows for sending workload traffic without encapsulation within the same AWS subnet.

", + "UserEditable": true, + "GoType": "*v3.AWSSrcDstCheckOption", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Egress gateway", + "Fields": [ + { + "Group": "Egress gateway", + "GroupWithSortPrefix": "70 Egress gateway", + "NameConfigFile": "EgressGatewayPollFailureCount", + "NameEnvVar": "FELIX_EgressGatewayPollFailureCount", + "NameYAML": "egressGatewayPollFailureCount", + "NameGoAPI": "EgressGatewayPollFailureCount", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "3", + "ParsedDefault": "3", + "ParsedDefaultJSON": "3", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "3", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The minimum number of poll failures before a remote Egress Gateway is considered\nto have failed.", + "DescriptionHTML": "

The minimum number of poll failures before a remote Egress Gateway is considered\nto have failed.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Egress gateway", + "GroupWithSortPrefix": "70 Egress gateway", + "NameConfigFile": "EgressGatewayPollInterval", + "NameEnvVar": "FELIX_EgressGatewayPollInterval", + "NameYAML": "egressGatewayPollInterval", + "NameGoAPI": "EgressGatewayPollInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "10", + "ParsedDefault": "10s", + "ParsedDefaultJSON": "10000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "10s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The interval at which Felix will poll remote egress gateways to check their\nhealth. Only Egress Gateways with a named \"health\" port will be polled in this way. Egress Gateways that\nfail the health check will be taken our of use as if they have been deleted.", + "DescriptionHTML": "

The interval at which Felix will poll remote egress gateways to check their\nhealth. Only Egress Gateways with a named \"health\" port will be polled in this way. Egress Gateways that\nfail the health check will be taken our of use as if they have been deleted.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Egress gateway", + "GroupWithSortPrefix": "70 Egress gateway", + "NameConfigFile": "EgressIPHostIfacePattern", + "NameEnvVar": "FELIX_EgressIPHostIfacePattern", + "NameYAML": "egressIPHostIfacePattern", + "NameGoAPI": "EgressIPHostIfacePattern", + "StringSchema": "Comma-delimited list of Linux interface names/regex patterns. Regex patterns must start/end with `/`.", + "StringSchemaHTML": "Comma-delimited list of Linux interface names/regex patterns. Regex patterns must start/end with /.", + "StringDefault": "", + "ParsedDefault": "[]", + "ParsedDefaultJSON": "null", + "ParsedType": "[]*regexp.Regexp", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "A comma-separated list of interface names which might send and receive egress traffic\nacross the cluster boundary, after it has left an Egress Gateway pod. Felix will ensure `src_valid_mark` sysctl flags\nare set correctly for matching interfaces.\nTo target multiple interfaces with a single string, the list supports regular expressions.\nFor regular expressions, wrap the value with `/`.\nExample: `/^bond/,eth0` will match all interfaces that begin with `bond` and also the interface `eth0`.", + "DescriptionHTML": "

A comma-separated list of interface names which might send and receive egress traffic\nacross the cluster boundary, after it has left an Egress Gateway pod. Felix will ensure src_valid_mark sysctl flags\nare set correctly for matching interfaces.\nTo target multiple interfaces with a single string, the list supports regular expressions.\nFor regular expressions, wrap the value with /.\nExample: /^bond/,eth0 will match all interfaces that begin with bond and also the interface eth0.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Egress gateway", + "GroupWithSortPrefix": "70 Egress gateway", + "NameConfigFile": "EgressIPRoutingRulePriority", + "NameEnvVar": "FELIX_EgressIPRoutingRulePriority", + "NameYAML": "egressIPRoutingRulePriority", + "NameGoAPI": "EgressIPRoutingRulePriority", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "100", + "ParsedDefault": "100", + "ParsedDefaultJSON": "100", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "100", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the priority value to use for the egress IP routing rule.", + "DescriptionHTML": "

Controls the priority value to use for the egress IP routing rule.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Egress gateway", + "GroupWithSortPrefix": "70 Egress gateway", + "NameConfigFile": "EgressIPSupport", + "NameEnvVar": "FELIX_EgressIPSupport", + "NameYAML": "egressIPSupport", + "NameGoAPI": "EgressIPSupport", + "StringSchema": "One of: `Disabled`, `EnabledPerNamespaceOrPerPod`, `EnabledPerNamespace`", + "StringSchemaHTML": "One of: Disabled, EnabledPerNamespaceOrPerPod, EnabledPerNamespace", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `EnabledPerNamespace`, `EnabledPerNamespaceOrPerPod`.", + "YAMLEnumValues": [ + "Disabled", + "EnabledPerNamespace", + "EnabledPerNamespaceOrPerPod" + ], + "YAMLSchemaHTML": "One of: Disabled, EnabledPerNamespace, EnabledPerNamespaceOrPerPod.", + "YAMLDefault": "Disabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Defines three different support modes for egress IP function. \n- Disabled: Egress IP function is disabled.\n- EnabledPerNamespace: Egress IP function is enabled and can be configured on a per-namespace basis;\n per-pod egress annotations are ignored.\n- EnabledPerNamespaceOrPerPod: Egress IP function is enabled and can be configured per-namespace or per-pod,\n with per-pod egress annotations overriding namespace annotations.", + "DescriptionHTML": "

Defines three different support modes for egress IP function. \n- Disabled: Egress IP function is disabled.\n- EnabledPerNamespace: Egress IP function is enabled and can be configured on a per-namespace basis;\n per-pod egress annotations are ignored.\n- EnabledPerNamespaceOrPerPod: Egress IP function is enabled and can be configured per-namespace or per-pod,\n with per-pod egress annotations overriding namespace annotations.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Egress gateway", + "GroupWithSortPrefix": "70 Egress gateway", + "NameConfigFile": "EgressIPVXLANPort", + "NameEnvVar": "FELIX_EgressIPVXLANPort", + "NameYAML": "egressIPVXLANPort", + "NameGoAPI": "EgressIPVXLANPort", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "4790", + "ParsedDefault": "4790", + "ParsedDefaultJSON": "4790", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "4790", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The port number of vxlan tunnel device for egress traffic.", + "DescriptionHTML": "

The port number of vxlan tunnel device for egress traffic.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Egress gateway", + "GroupWithSortPrefix": "70 Egress gateway", + "NameConfigFile": "EgressIPVXLANVNI", + "NameEnvVar": "FELIX_EgressIPVXLANVNI", + "NameYAML": "egressIPVXLANVNI", + "NameGoAPI": "EgressIPVXLANVNI", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "4097", + "ParsedDefault": "4097", + "ParsedDefaultJSON": "4097", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "4097", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The VNI ID of vxlan tunnel device for egress traffic.", + "DescriptionHTML": "

The VNI ID of vxlan tunnel device for egress traffic.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "External network support", + "Fields": [ + { + "Group": "External network support", + "GroupWithSortPrefix": "70 External network support", + "NameConfigFile": "ExternalNetworkRoutingRulePriority", + "NameEnvVar": "FELIX_ExternalNetworkRoutingRulePriority", + "NameYAML": "externalNetworkRoutingRulePriority", + "NameGoAPI": "ExternalNetworkRoutingRulePriority", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "102", + "ParsedDefault": "102", + "ParsedDefaultJSON": "102", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "102", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the priority value to use for the external network routing rule.", + "DescriptionHTML": "

Controls the priority value to use for the external network routing rule.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "External network support", + "GroupWithSortPrefix": "70 External network support", + "NameConfigFile": "ExternalNetworkSupport", + "NameEnvVar": "FELIX_ExternalNetworkSupport", + "NameYAML": "externalNetworkSupport", + "NameGoAPI": "ExternalNetworkSupport", + "StringSchema": "One of: `Disabled`, `Enabled`", + "StringSchemaHTML": "One of: Disabled, Enabled", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `Enabled`.", + "YAMLEnumValues": [ + "Disabled", + "Enabled" + ], + "YAMLSchemaHTML": "One of: Disabled, Enabled.", + "YAMLDefault": "Disabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Defines two different support modes for external network function. \n- Disabled: External network function is disabled.\n- Enabled: External network function is enabled.", + "DescriptionHTML": "

Defines two different support modes for external network function. \n- Disabled: External network function is disabled.\n- Enabled: External network function is enabled.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Packet capture", + "Fields": [ + { + "Group": "Packet capture", + "GroupWithSortPrefix": "80 Packet capture", + "NameConfigFile": "CaptureDir", + "NameEnvVar": "FELIX_CaptureDir", + "NameYAML": "captureDir", + "NameGoAPI": "CaptureDir", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "/var/log/calico/pcap", + "ParsedDefault": "/var/log/calico/pcap", + "ParsedDefaultJSON": "\"/var/log/calico/pcap\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "/var/log/calico/pcap", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls directory to store file capture.", + "DescriptionHTML": "

Controls directory to store file capture.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Packet capture", + "GroupWithSortPrefix": "80 Packet capture", + "NameConfigFile": "CaptureMaxFiles", + "NameEnvVar": "FELIX_CaptureMaxFiles", + "NameYAML": "captureMaxFiles", + "NameGoAPI": "CaptureMaxFiles", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "2", + "ParsedDefault": "2", + "ParsedDefaultJSON": "2", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "2", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls number of rotated capture file to keep.", + "DescriptionHTML": "

Controls number of rotated capture file to keep.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Packet capture", + "GroupWithSortPrefix": "80 Packet capture", + "NameConfigFile": "CaptureMaxSizeBytes", + "NameEnvVar": "FELIX_CaptureMaxSizeBytes", + "NameYAML": "captureMaxSizeBytes", + "NameGoAPI": "CaptureMaxSizeBytes", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "10000000", + "ParsedDefault": "10000000", + "ParsedDefaultJSON": "10000000", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "10000000", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the max size of a file capture.", + "DescriptionHTML": "

Controls the max size of a file capture.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Packet capture", + "GroupWithSortPrefix": "80 Packet capture", + "NameConfigFile": "CaptureRotationSeconds", + "NameEnvVar": "FELIX_CaptureRotationSeconds", + "NameYAML": "captureRotationSeconds", + "NameGoAPI": "CaptureRotationSeconds", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "3600", + "ParsedDefault": "3600", + "ParsedDefaultJSON": "3600", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "3600", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Controls the time rotation of a packet capture.", + "DescriptionHTML": "

Controls the time rotation of a packet capture.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "L7 proxy", + "Fields": [ + { + "Group": "L7 proxy", + "GroupWithSortPrefix": "90 L7 proxy", + "NameConfigFile": "TPROXYMode", + "NameEnvVar": "FELIX_TPROXYMode", + "NameYAML": "tproxyMode", + "NameGoAPI": "TPROXYMode", + "StringSchema": "One of: `Disabled`, `EnabledAllServices`, `Enabled`", + "StringSchemaHTML": "One of: Disabled, EnabledAllServices, Enabled", + "StringDefault": "Disabled", + "ParsedDefault": "Disabled", + "ParsedDefaultJSON": "\"Disabled\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "One of: `Disabled`, `Enabled`, `EnabledAllServices`.", + "YAMLEnumValues": [ + "Disabled", + "Enabled", + "EnabledAllServices" + ], + "YAMLSchemaHTML": "One of: Disabled, Enabled, EnabledAllServices.", + "YAMLDefault": "Disabled", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets whether traffic is directed through a transparent proxy\nfor further processing or not and how is the proxying done.", + "DescriptionHTML": "

Sets whether traffic is directed through a transparent proxy\nfor further processing or not and how is the proxying done.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "L7 proxy", + "GroupWithSortPrefix": "90 L7 proxy", + "NameConfigFile": "TPROXYPort", + "NameEnvVar": "FELIX_TPROXYPort", + "NameYAML": "tproxyPort", + "NameGoAPI": "TPROXYPort", + "StringSchema": "Integer", + "StringSchemaHTML": "Integer", + "StringDefault": "16001", + "ParsedDefault": "16001", + "ParsedDefaultJSON": "16001", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer", + "YAMLDefault": "16001", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Sets to which port proxied traffic should be redirected.", + "DescriptionHTML": "

Sets to which port proxied traffic should be redirected.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "L7 proxy", + "GroupWithSortPrefix": "90 L7 proxy", + "NameConfigFile": "TPROXYUpstreamConnMark", + "NameEnvVar": "FELIX_TPROXYUpstreamConnMark", + "NameYAML": "tproxyUpstreamConnMark", + "NameGoAPI": "TPROXYUpstreamConnMark", + "StringSchema": "32-bit bitmask (hex or deccimal allowed) with at least 2 bits set, example: `0xffff0000`", + "StringSchemaHTML": "32-bit bitmask (hex or deccimal allowed) with at least 2 bits set, example: 0xffff0000", + "StringDefault": "0x17", + "ParsedDefault": "23", + "ParsedDefaultJSON": "23", + "ParsedType": "uint32", + "YAMLType": "integer", + "YAMLSchema": "Unsigned 32-bit integer.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Unsigned 32-bit integer.", + "YAMLDefault": "0x17", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Tells Felix which mark is used by the proxy for its upstream\nconnections so that Felix can program the dataplane correctly.", + "DescriptionHTML": "

Tells Felix which mark is used by the proxy for its upstream\nconnections so that Felix can program the dataplane correctly.

", + "UserEditable": true, + "GoType": "*uint32", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Debug/test-only (generally unsupported)", + "Fields": [ + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugBPFCgroupV2", + "NameEnvVar": "FELIX_DebugBPFCgroupV2", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "LocalOnly", + "Description": "Controls the cgroup v2 path that we apply the connect-time load balancer to. Most distros\nare configured for cgroup v1, which prevents all but the root cgroup v2 from working so this is only useful\nfor development right now.", + "DescriptionHTML": "

Controls the cgroup v2 path that we apply the connect-time load balancer to. Most distros\nare configured for cgroup v1, which prevents all but the root cgroup v2 from working so this is only useful\nfor development right now.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugCPUProfilePath", + "NameEnvVar": "FELIX_DebugCPUProfilePath", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file", + "StringSchemaHTML": "Path to file", + "StringDefault": "/tmp/felix-cpu-.pprof", + "ParsedDefault": "/tmp/felix-cpu-.pprof", + "ParsedDefaultJSON": "\"/tmp/felix-cpu-\\u003ctimestamp\\u003e.pprof\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`.", + "DescriptionHTML": "

Unsupported diagnostic setting, used when testing Felix. Not exposed in FelixConfiguration.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugCloudWatchLogsFile", + "NameEnvVar": "FELIX_DebugCloudWatchLogsFile", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Path to file", + "StringSchemaHTML": "Path to file", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`.", + "DescriptionHTML": "

Unsupported diagnostic setting, used when testing Felix. Not exposed in FelixConfiguration.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugConsoleEnabled", + "NameEnvVar": "FELIX_DebugConsoleEnabled", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`.", + "DescriptionHTML": "

Unsupported diagnostic setting, used when testing Felix. Not exposed in FelixConfiguration.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugDNSDoNotWriteIPSets", + "NameEnvVar": "FELIX_DebugDNSDoNotWriteIPSets", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`.", + "DescriptionHTML": "

Unsupported diagnostic setting, used when testing Felix. Not exposed in FelixConfiguration.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugDNSResponseDelay", + "NameEnvVar": "FELIX_DebugDNSResponseDelay", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Milliseconds (floating point)", + "StringSchemaHTML": "Milliseconds (floating point)", + "StringDefault": "0", + "ParsedDefault": "0s", + "ParsedDefaultJSON": "0", + "ParsedType": "time.Duration", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`.", + "DescriptionHTML": "

Unsupported diagnostic setting, used when testing Felix. Not exposed in FelixConfiguration.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugDisableLogDropping", + "NameEnvVar": "FELIX_DebugDisableLogDropping", + "NameYAML": "debugDisableLogDropping", + "NameGoAPI": "DebugDisableLogDropping", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "false", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Disables the dropping of log messages when the log buffer is full. This can\nsignificantly impact performance if log write-out is a bottleneck.", + "DescriptionHTML": "

Disables the dropping of log messages when the log buffer is full. This can\nsignificantly impact performance if log write-out is a bottleneck.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugHost", + "NameEnvVar": "FELIX_DebugHost", + "NameYAML": "debugHost", + "NameGoAPI": "DebugHost", + "StringSchema": "String matching regex `^[a-zA-Z0-9:._+-]{1,64}$`", + "StringSchemaHTML": "String matching regex ^[a-zA-Z0-9:._+-]{1,64}$", + "StringDefault": "localhost", + "ParsedDefault": "localhost", + "ParsedDefaultJSON": "\"localhost\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "localhost", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The host IP or hostname to bind the debug port to. Only used\nif DebugPort is set.", + "DescriptionHTML": "

The host IP or hostname to bind the debug port to. Only used\nif DebugPort is set.

", + "UserEditable": true, + "GoType": "*string", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugMemoryProfilePath", + "NameEnvVar": "FELIX_DebugMemoryProfilePath", + "NameYAML": "debugMemoryProfilePath", + "NameGoAPI": "DebugMemoryProfilePath", + "StringSchema": "Path to file", + "StringSchemaHTML": "Path to file", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "The path to write the memory profile to when triggered by signal.", + "DescriptionHTML": "

The path to write the memory profile to when triggered by signal.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugPanicAfter", + "NameEnvVar": "FELIX_DebugPanicAfter", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "0", + "ParsedDefault": "0s", + "ParsedDefaultJSON": "0", + "ParsedType": "time.Duration", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`.", + "DescriptionHTML": "

Unsupported diagnostic setting, used when testing Felix. Not exposed in FelixConfiguration.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugPort", + "NameEnvVar": "FELIX_DebugPort", + "NameYAML": "debugPort", + "NameGoAPI": "DebugPort", + "StringSchema": "Integer: [0,65535]", + "StringSchemaHTML": "Integer: [0,65535]", + "StringDefault": "", + "ParsedDefault": "0", + "ParsedDefaultJSON": "0", + "ParsedType": "int", + "YAMLType": "integer", + "YAMLSchema": "Integer: [0,65535]", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Integer: [0,65535]", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "If set, enables Felix's debug HTTP port, which allows memory and CPU profiles\nto be retrieved. The debug port is not secure, it should not be exposed to the internet.", + "DescriptionHTML": "

If set, enables Felix's debug HTTP port, which allows memory and CPU profiles\nto be retrieved. The debug port is not secure, it should not be exposed to the internet.

", + "UserEditable": true, + "GoType": "*int", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugSimulateCalcGraphHangAfter", + "NameEnvVar": "FELIX_DebugSimulateCalcGraphHangAfter", + "NameYAML": "debugSimulateCalcGraphHangAfter", + "NameGoAPI": "DebugSimulateCalcGraphHangAfter", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "0", + "ParsedDefault": "0s", + "ParsedDefaultJSON": "0", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to simulate a hang in the calculation graph after the specified duration.\nThis is useful in tests of the watchdog system only!", + "DescriptionHTML": "

Used to simulate a hang in the calculation graph after the specified duration.\nThis is useful in tests of the watchdog system only!

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugSimulateDataRace", + "NameEnvVar": "FELIX_DebugSimulateDataRace", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`.", + "DescriptionHTML": "

Unsupported diagnostic setting, used when testing Felix. Not exposed in FelixConfiguration.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugSimulateDataplaneApplyDelay", + "NameEnvVar": "FELIX_DebugSimulateDataplaneApplyDelay", + "NameYAML": "debugSimulateDataplaneApplyDelay", + "NameGoAPI": "DebugSimulateDataplaneApplyDelay", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "0", + "ParsedDefault": "0s", + "ParsedDefaultJSON": "0", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Adds an artificial delay to every dataplane operation. This is useful for\nsimulating a heavily loaded system for test purposes only.", + "DescriptionHTML": "

Adds an artificial delay to every dataplane operation. This is useful for\nsimulating a heavily loaded system for test purposes only.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugSimulateDataplaneHangAfter", + "NameEnvVar": "FELIX_DebugSimulateDataplaneHangAfter", + "NameYAML": "debugSimulateDataplaneHangAfter", + "NameGoAPI": "DebugSimulateDataplaneHangAfter", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "0", + "ParsedDefault": "0s", + "ParsedDefaultJSON": "0", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Used to simulate a hang in the dataplane after the specified duration.\nThis is useful in tests of the watchdog system only!", + "DescriptionHTML": "

Used to simulate a hang in the dataplane after the specified duration.\nThis is useful in tests of the watchdog system only!

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugUseShortPollIntervals", + "NameEnvVar": "FELIX_DebugUseShortPollIntervals", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "false", + "ParsedDefault": "false", + "ParsedDefaultJSON": "false", + "ParsedType": "bool", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`.", + "DescriptionHTML": "

Unsupported diagnostic setting, used when testing Felix. Not exposed in FelixConfiguration.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "DebugWindowsPktMonStartArgs", + "NameEnvVar": "FELIX_DebugWindowsPktMonStartArgs", + "NameYAML": "", + "NameGoAPI": "", + "StringSchema": "String", + "StringSchemaHTML": "String", + "StringDefault": "", + "ParsedDefault": "", + "ParsedDefaultJSON": "\"\"", + "ParsedType": "string", + "YAMLType": "", + "YAMLSchema": "", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "", + "YAMLDefault": "", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Unsupported diagnostic setting, used when testing Felix. Not exposed in `FelixConfiguration`.", + "DescriptionHTML": "

Unsupported diagnostic setting, used when testing Felix. Not exposed in FelixConfiguration.

", + "UserEditable": true, + "GoType": "", + "OpenSourceOnly": false + }, + { + "Group": "Debug/test-only (generally unsupported)", + "GroupWithSortPrefix": "97 Debug/test-only (generally unsupported)", + "NameConfigFile": "StatsDumpFilePath", + "NameEnvVar": "FELIX_StatsDumpFilePath", + "NameYAML": "statsDumpFilePath", + "NameGoAPI": "StatsDumpFilePath", + "StringSchema": "Path to file", + "StringSchemaHTML": "Path to file", + "StringDefault": "/var/log/calico/stats/dump", + "ParsedDefault": "/var/log/calico/stats/dump", + "ParsedDefaultJSON": "\"/var/log/calico/stats/dump\"", + "ParsedType": "string", + "YAMLType": "string", + "YAMLSchema": "String.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "String.", + "YAMLDefault": "/var/log/calico/stats/dump", + "Required": false, + "OnParseFailure": "Exit", + "AllowedConfigSources": "All", + "Description": "The path to write a diagnostic flow logs statistics dump to when triggered by signal.", + "DescriptionHTML": "

The path to write a diagnostic flow logs statistics dump to when triggered by signal.

", + "UserEditable": true, + "GoType": "string", + "OpenSourceOnly": false + } + ] + }, + { + "Name": "Usage reporting", + "Fields": [ + { + "Group": "Usage reporting", + "GroupWithSortPrefix": "99 Usage reporting", + "NameConfigFile": "UsageReportingEnabled", + "NameEnvVar": "FELIX_UsageReportingEnabled", + "NameYAML": "usageReportingEnabled", + "NameGoAPI": "UsageReportingEnabled", + "StringSchema": "Boolean: `true`, `1`, `yes`, `y`, `t` accepted as True; `false`, `0`, `no`, `n`, `f` accepted (case insensitively) as False.", + "StringSchemaHTML": "Boolean: true, 1, yes, y, t accepted as True; false, 0, no, n, f accepted (case insensitively) as False.", + "StringDefault": "true", + "ParsedDefault": "true", + "ParsedDefaultJSON": "true", + "ParsedType": "bool", + "YAMLType": "boolean", + "YAMLSchema": "Boolean.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Boolean.", + "YAMLDefault": "true", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Unused in Calico Enterprise, usage reporting is permanently disabled.", + "DescriptionHTML": "

Unused in Calico Enterprise, usage reporting is permanently disabled.

", + "UserEditable": true, + "GoType": "*bool", + "OpenSourceOnly": true + }, + { + "Group": "Usage reporting", + "GroupWithSortPrefix": "99 Usage reporting", + "NameConfigFile": "UsageReportingInitialDelaySecs", + "NameEnvVar": "FELIX_UsageReportingInitialDelaySecs", + "NameYAML": "usageReportingInitialDelay", + "NameGoAPI": "UsageReportingInitialDelay", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "300", + "ParsedDefault": "5m0s", + "ParsedDefaultJSON": "300000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "5m0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Unused in Calico Enterprise, usage reporting is permanently disabled.", + "DescriptionHTML": "

Unused in Calico Enterprise, usage reporting is permanently disabled.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": true + }, + { + "Group": "Usage reporting", + "GroupWithSortPrefix": "99 Usage reporting", + "NameConfigFile": "UsageReportingIntervalSecs", + "NameEnvVar": "FELIX_UsageReportingIntervalSecs", + "NameYAML": "usageReportingInterval", + "NameGoAPI": "UsageReportingInterval", + "StringSchema": "Seconds (floating point)", + "StringSchemaHTML": "Seconds (floating point)", + "StringDefault": "86400", + "ParsedDefault": "24h0m0s", + "ParsedDefaultJSON": "86400000000000", + "ParsedType": "time.Duration", + "YAMLType": "string", + "YAMLSchema": "Duration string, for example `1m30s123ms` or `1h5m`.", + "YAMLEnumValues": null, + "YAMLSchemaHTML": "Duration string, for example 1m30s123ms or 1h5m.", + "YAMLDefault": "24h0m0s", + "Required": false, + "OnParseFailure": "ReplaceWithDefault", + "AllowedConfigSources": "All", + "Description": "Unused in Calico Enterprise, usage reporting is permanently disabled.", + "DescriptionHTML": "

Unused in Calico Enterprise, usage reporting is permanently disabled.

", + "UserEditable": true, + "GoType": "*v1.Duration", + "OpenSourceOnly": true + } + ] + } + ] +} diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/index.js b/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/index.js new file mode 100644 index 0000000000..2c2ca6e2c7 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/index.js @@ -0,0 +1,108 @@ +import React, { useState, useEffect } from 'react'; +import jsonData from './config-params.json'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import TableResource from './TableResource'; +import TableConfig from './TableConfig'; +import TableEnv from './TableEnv'; +import styles from './styles.module.css'; + +const FelixConfigTable = ({ configType, fieldData }) => { + let table; + + switch (configType) { + case 'yaml': + table = ; + break; + case 'config': + table = ; + break; + case 'env': + table = ; + break; + default: + table =

Unknown config type

; + } + + return
{table}
; +}; + +const FelixConfig = ({ configType, name }) => { + const matchedGroup = jsonData.Groups.find((group) => group.Name === name); + // Debugging logs + + if (!matchedGroup) { + // Fail the build loudly rather than silently dropping the group's fields. + // `name` must exactly match a group Name in config-params.json, which is + // auto-synced from the Felix source repos. A prose/casing sweep once + // restyled these lookup keys and silently dropped fields (DOCS-2970). + throw new Error( + `FelixConfig: no group named '${name}' in config-params.json. The 'name' ` + + `prop must exactly match a group Name in config-params.json; do not ` + + `restyle it as prose.` + ); + } + + let content; + + if (configType === 'yaml') { + content = ( +
+ {matchedGroup.Fields.map((field, index) => ( + field.NameYAML ? ( +
+
+ {field.NameYAML} +
+ +
+ ) : null + ))} +
+ ); + } else if (configType === 'configenv') { + content = ( +
+ {matchedGroup.Fields.map((field, index) => ( +
+

{field.NameConfigFile}

+ + + + + + + + +
+ ))} +
+ ); + } else { + content = ( +
+

You got it wrong

+
+ ); + } + return
{content}
; +}; + +export default FelixConfig; diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/styles.module.css b/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/styles.module.css new file mode 100644 index 0000000000..d831a8ed05 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/components/FelixConfig/styles.module.css @@ -0,0 +1,12 @@ +.felixTable { + width: 100%; + display: table; +} + +.felixTable td p:last-child { + margin-bottom: 0; +} + +.tabsContainer { + margin-bottom: 60px; +} diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/components/ReqsKernel.js b/calico-cloud_versioned_docs/version-24-1/_includes/components/ReqsKernel.js new file mode 100644 index 0000000000..124f85b24c --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/components/ReqsKernel.js @@ -0,0 +1,69 @@ +import React from 'react'; +import Admonition from '@theme/Admonition'; +import Heading from '@theme/Heading'; + +import { prodname } from '../../variables'; + +export default function ReqsKernel() { + return ( + <> + + Kernel dependencies + + +

If you are using one of the recommended distributions, you will already satisfy these.

+ +

+ Due to the large number of distributions and kernel version out there, it’s hard to be precise about the names + of the particular kernel modules that are required to run {prodname}. However, in general, you’ll need: +

+
    +
  • +

    + The iptables modules (both the “legacy” and “nft” variants are supported). These are typically + broken up into many small modules, one for each type of match criteria and one for each type of action.{' '} + {prodname} requires: +

    +
      +
    • The “base” modules (including the IPv6 versions if IPv6 is enabled in your cluster).
      • +
    • + At least the following match criteria: set, rpfilter, addrtype,{' '} + comment, conntrack, icmp, tcp, udp,{' '} + ipvs, icmpv6 (if IPv6 is enabled in your kernel), mark,{' '} + multiport, rpfilter, sctp, ipvs (if using + kube-proxy in IPVS mode). +
      • +
    • + At least the following actions: REJECT, ACCEPT, DROP,{' '} + LOG. +
      • +
    +
    • +
  • +

    IP sets support.

    +
    • +
  • +

    Netfilter Conntrack support compiled in (with SCTP support if using SCTP).

    +
    • +
  • +

    + IPVS support if using kube-proxy in IPVS mode. +

    +
    • +
  • +

    + IPIP, VXLAN, Wireguard support, if using {prodname} networking in one of those modes. +

    +
    • +
  • +

    + eBPF (including the tc hook support) and XDP (if you want to use the eBPF data plane). +

    +
    • +
+ + ); +} diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/components/ReqsSys.js b/calico-cloud_versioned_docs/version-24-1/_includes/components/ReqsSys.js new file mode 100644 index 0000000000..07da9c3de2 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/components/ReqsSys.js @@ -0,0 +1,434 @@ +import React from 'react'; + +import Admonition from '@theme/Admonition'; +import Link from '@docusaurus/Link'; +import Heading from '@theme/Heading'; + +import { orchestrators } from '@site/variables'; +import { prodname, baseUrl } from '../../variables'; + +function NodeRequirementsEnt(props) { + return ( + <> + + Node requirements + +
    +
  • +

    x86-64 processor with at least 2 cores, 8.0GB RAM and 20 GB free disk space

    +
    • +
  • +

    + Linux kernel 5.10 or later with required dependencies. The + following distributions have the required kernel, its dependencies, and are known to work well with{' '} + {prodname} and {props.orch}. +

    +
      + {(props.orch === orchestrators.Kubernetes || props.orch === orchestrators.HostProtection) && ( + <> +
    • CentOS 8
      • +
    • Ubuntu 18.04 and 20.04
      • +
    • RHEL 7 and 8
      • +
    • Debian 10
      • + + )} + {props.orch === orchestrators.OpenShift && ( + <> +
    • Red Hat Enterprise Linux CoreOS
      • + + )} + {props.orch === orchestrators.OpenStack && ( + <> +
    • Ubuntu 18.04 and 20.04
      • +
    • CentOS 8
      • + + )} +
    +
    • +
  • +

    + {prodname} must be able to manage cali* + interfaces on the host. When IPIP is enabled (the default), + {prodname} also needs to be able to manage tunl* + interfaces. When VXLAN is enabled, {prodname} also needs to be able to manage the vxlan.calico{' '} + interface. +

    + {/*} + +

    + Many Linux distributions, such as most of the above, include NetworkManager. By default, NetworkManager + does not allow + {prodname} to manage interfaces. If your nodes have NetworkManager, complete the steps in{' '} + + Preventing NetworkManager from controlling {prodname} interfaces + {' '} + before installing {prodname}. +

    +     + */} +
    • +
  • +

    + If your Linux distribution comes with installed Firewalld or another iptables manager it should be disabled. + These may interfere with rules added by {prodname} and result in unexpected behavior. +

    + +

    + If a host firewall is needed, it can be configured by {prodname} HostEndpoint and GlobalNetworkPolicy. + More information about configuration at Security for host. +

    +     +
    • +
  • +

    + In order to properly run Elasticsearch, nodes must be configured according to the{' '} + + Elasticsearch system configuration documentation. + +

    +
    • +
  • +

    + The Typha autoscaler requires a minimum number of Linux worker nodes based on total number of schedulable + nodes. +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Total schedulable nodesRequired Linux nodes for Typha replicas
    11
    22
    33
    up to 2504
    up to 5005
    up to 10006
    up to 15007
    up to 20008
    2000 or more10
    +
    • +
+ + ); +} + +function KeyValueStore(props) { + return ( + <> + + Datastore requirements + +

+ {prodname} requires a key/value store accessible by all {prodname} components.  + { + { + OpenShift: With OpenShift, the Kubernetes API datastore is used for the key/value store., + Kubernetes: ( + + On Kubernetes, you can configure {prodname} to access an etcdv3 cluster directly or to use the + Kubernetes API datastore. + + ), + OpenStack: ( + + For production you will likely want multiple nodes for greater performance and reliability. If you don’t + already have an etcdv3 cluster to connect to, please refer to{' '} + the upstream etcd docs for detailed advice and setup. + + ), + 'host protection': The key/value store must be etcdv3., + }[props.orch] + } +

+ + ); +} + +function NetworkRequirementsEnt(props) { + return ( + <> + + Network requirements + +

+ Ensure that your hosts and firewalls allow the necessary traffic based on your configuration. +

+ {/*Above TODO-XREFS-CC */} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {props.orch === orchestrators.Kubernetes && ( + <> + + + + + + + + + + + + )} + {props.orch === orchestrators.OpenShift && ( + <> + + + + + + + + + + + + )} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ConfigurationHost(s)Port/protocol
+ {prodname} networking options + IP-in-IP (default)Protocol number 4
BGPTCP 179
VXLANUDP 4789
WireguardUDP 51820 (default)
IPv6 WireguardUDP 51821 (default)
+ Cluster scaling + Any {prodname} networking option above with Typha agents enabledTCP 5473 (default)
+ APIs + Kubernetes API (kube-apiserver) to access Kubernetes API datastoreOften TCP 443 or 6443*
{prodname} API serverTCP 8080 and 5443 (default)
+ APIs + Kubernetes API (kube-apiserver) to access Kubernetes API datastoreOften TCP 443 or 8443*
{prodname} API serverTCP 8080 and 5443 (default)
+ Nodes + calico-node (Felix, BIRD, confd)TCP 9090 (default)
+ Component metrics + Prometheus metricsTCP 9081 (default)
Prometheus BGP metricsTCP 9900 (default)
Prometheus API serviceTCP 9090 (default)
Prometheus AlertmanagerTCP 9093 (default)
+ Logs and storage + Elasticsearch with fluentd datastoreTCP 9200 (default)
Elasticssearch for cloud (ECK)TCP 9443 (default)
Elasticsearch gatewayTCP 5444 (default)
+ Visibility and troubleshooting + KibanaTCP 5601 (default)
Packet capture APITCP 8444 (default)
{prodname} Manager UITCP 9443 (default)
+ Intrusion Detection System (IDS) + {prodname} intrusion detectionTCP 5443 (default)
+ Multi-cluster management + Additional port required for Manager UITCP 9449
+ Egress gateway + {prodname} egress gatewayUDP 4790
+ {(props.orch === orchestrators.Kubernetes || props.orch === orchestrators.OpenShift) && ( + <> +

+ *{' '} + + The value passed to kube-apiserver using the --secure-port + flag. If you cannot locate this, check the targetPort value returned by   + kubectl get svc kubernetes -o yaml. + +

+ + )} + {props.orch === orchestrators.OpenStack && ( +

+ *{' '} + + If your compute hosts connect directly and don’t use IP-in-IP, you don’t need to allow IP-in-IP traffic. + +

+ )} + + ); +} + +function Privileges(props) { + return ( + <> + + Privilege requirements + +

+ Ensure that {prodname} has the CAP_SYS_ADMIN privilege. +

+

+ The simplest way to provide the necessary privilege is to run {prodname} as root or in a privileged container. +

+ {props.orch === orchestrators.Kubernetes && ( + <> +

+ When installed as a Kubernetes daemon set, {prodname} meets this requirement by running as a privileged + container. This requires that the kubelet be allowed to run privileged containers. There are two ways this + can be achieved. +

+
    +
  • + Specify --allow-privileged on the kubelet (deprecated). +
    • +
  • + Use a{' '} + pod security policy. +
    • +
+ + )} + + ); +} + +export default function ReqsSys(props) { + return ( + <> + + + + + + ); +} diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_create-kubeconfig.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_create-kubeconfig.mdx new file mode 100644 index 0000000000..7b47173e65 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_create-kubeconfig.mdx @@ -0,0 +1,40 @@ +1. Create a service account + + ```bash + SA_NAME=my-host + kubectl create serviceaccount $SA_NAME -n calico-system -o yaml + ``` + +1. Obtain the token for the secret associated with your host + + ```bash + kubectl describe secret -n calico-system $(kubectl get serviceaccount -n calico-system $SA_NAME -o=jsonpath="{.secrets[0].name}") + ``` + +1. Use a text editor to create a kubeconfig file + + ```yaml + apiVersion: v1 + kind: Config + + users: + - name: my-host + user: + token: + + clusters: + - cluster: + certificate-authority-data: + server: + name: + + contexts: + - context: + cluster: my-cluster + user: my-host + name: my-host + + current-context: my-cluster + ``` + + Take the cluster information from an already existing kubeconfig. diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_determine-ipam.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_determine-ipam.mdx new file mode 100644 index 0000000000..f44121d82a --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_determine-ipam.mdx @@ -0,0 +1,9 @@ +If you are not sure which IPAM your cluster is using, the way to tell depends on install method. + +The IPAM plugin can be queried on the default Installation resource. + +``` +kubectl get installation default -o go-template --template {{.spec.cni.ipam.type}} +``` + +If your cluster is using Calico IPAM, the above command should return a result of `Calico`. diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_docker-container-service.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_docker-container-service.mdx new file mode 100644 index 0000000000..afc446d97e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_docker-container-service.mdx @@ -0,0 +1,70 @@ +import NonClusterReadOnlyStep from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_non-cluster-read-only-step.mdx'; +import EnvironmentFile from '@site/calico-cloud_versioned_docs/version-24-1/_includes/components/EnvironmentFile'; + +This section describes how to run `$[nodecontainer]` as a Docker container. + + + +### Step 2: Create environment file + + + +### Step 3: Configure the init system + +Use an init daemon (like systemd or upstart) to start the $[nodecontainer] image as a service using the EnvironmentFile values. + +Sample systemd service file: `$[noderunning].service` + +```shell +[Unit] +Description=$[noderunning] +After=docker.service +Requires=docker.service + +[Service] +EnvironmentFile=/etc/calico/calico.env +ExecStartPre=-/usr/bin/docker rm -f $[noderunning] +ExecStart=/usr/bin/docker run --net=host --privileged \ + --name=$[noderunning] \ + -e NODENAME=${CALICO_NODENAME} \ + -e IP=${CALICO_IP} \ + -e IP6=${CALICO_IP6} \ + -e CALICO_NETWORKING_BACKEND=${CALICO_NETWORKING_BACKEND} \ + -e AS=${CALICO_AS} \ + -e NO_DEFAULT_POOLS=${NO_DEFAULT_POOLS} \ + -e DATASTORE_TYPE=${DATASTORE_TYPE} \ + -e KUBECONFIG=${KUBECONFIG} \ + -v /var/log/calico:/var/log/calico \ + -v /var/lib/calico:/var/lib/calico \ + -v /var/run/calico:/var/run/calico \ + -v /run/docker/plugins:/run/docker/plugins \ + -v /lib/modules:/lib/modules \ + -v /etc/pki:/pki \ + $[registry]$[componentImage.cnxNode] /bin/calico-node -felix + +ExecStop=-/usr/bin/docker stop $[noderunning] + +Restart=on-failure +StartLimitBurst=3 +StartLimitInterval=60s + +[Install] +WantedBy=multi-user.target +``` + +Upon start, the systemd service: + +- Confirms Docker is installed under the `[Unit]` section +- Gets environment variables from the environment file above +- Removes existing `$[nodecontainer]` container (if it exists) +- Starts `$[nodecontainer]` + +The script also stops the `$[nodecontainer]` container when the service is stopped. + +:::note + +Depending on how you've installed Docker, the name of the Docker service +under the `[Unit]` section may be different (such as `docker-engine.service`). +Be sure to check this before starting the service. + +::: diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_domain-names.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_domain-names.mdx new file mode 100644 index 0000000000..bf21e94ec9 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_domain-names.mdx @@ -0,0 +1,23 @@ +When a configured domain name has no wildcard (`*`), it matches exactly that domain name. For example: + +- `microsoft.com` +- `tigera.io` + +With a single asterisk in any part of the domain name, it matches 1 or more path components at that position. For example: + +- `*.google.com` matches `www.google.com` and `www.ipv6.google.com`, but not `google.com` +- `www.*.com` matches `www.sun.com` and `www.apple.com`, but not `www.com` +- `update.*.mycompany.com` matches `update.tools.mycompany.com`, `update.secure.suite.mycompany.com`, and so on + +**Not** supported are: + +- Multiple wildcards in the same domain, for example: `*.*.mycompany.com` +- Asterisks that are not the entire component, for example: `www.g*.com` +- A wildcard as the last component, for example: `www.mycompany.*` +- More general wildcards, such as regular expressions + +:::note + +The `Inline` DNS policy mode only supports prefix wildcards like `*.example.com`. Non-prefix wildcards (for example, `www.*.com` or `update.*.mycompany.com`) are not matched in `Inline` mode and require `DelayDeniedPacket` or `NoDelay` mode. `Inline` is available for the eBPF data plane (`BPFDNSPolicyMode: Inline`, the default on kernels 5.17+ / RedHat 5.14+) and for the iptables data plane (`DNSPolicyMode: Inline`). See [DNSPolicyMode](../../reference/resources/felixconfig.mdx#dnspolicymode) and [BPFDNSPolicyMode](../../reference/resources/felixconfig.mdx#bpfdnspolicymode). + +::: diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_ebpf-value.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_ebpf-value.mdx new file mode 100644 index 0000000000..fe2072d88f --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_ebpf-value.mdx @@ -0,0 +1,12 @@ +The eBPF data plane mode has several advantages over standard Linux networking pipeline mode: + +- It scales to higher throughput. +- It uses less CPU per GBit. +- It has native support for Kubernetes services (without needing kube-proxy) that: + + - Reduces first packet latency for packets to services. + - Preserves external client source IP addresses all the way to the pod. + - Supports DSR (Direct Server Return) for more efficient service routing. + - Uses less CPU than kube-proxy to keep the data plane in sync. + +To learn more and see performance metrics from our test environment, see the blog, [Introducing the Calico eBPF data plane](https://www.projectcalico.org/introducing-the-calico-ebpf-dataplane/). diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_endpointport.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_endpointport.mdx new file mode 100644 index 0000000000..05dde33612 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_endpointport.mdx @@ -0,0 +1,15 @@ +An EndpointPort associates a name with a particular TCP/UDP/SCTP port of the endpoint, allowing it to +be referenced as a named port in [policy rules](../../reference/resources/networkpolicy.mdx#entityrule). + +| Field | Description | Accepted Values | Schema | Default | +| -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------- | ------ | ------- | +| name | The name to attach to this port, allowing it to be referred to in [policy rules](../../reference/resources/networkpolicy.mdx#entityrule). Names must be unique within an endpoint. | | string | | +| protocol | The protocol of this named port. | `TCP`, `UDP`, `SCTP` | string | | +| port | The workload port number. | `1`-`65535` | int | | + +:::note + +On their own, EndpointPort entries don't result in any change to the connectivity of the port. +They only have an effect if they are referred to in policy. + +::: diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_entityrule.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_entityrule.mdx new file mode 100644 index 0000000000..06adee8c36 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_entityrule.mdx @@ -0,0 +1,101 @@ +import DomainNames from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_domain-names.mdx'; + +Entity rules specify the attributes of the source or destination of a packet that must match for the rule as a whole +to match. Packets can be matched on combinations of: + +- Identity of the source/destination, by using [Selectors](#selectors) or by specifying a particular + Kubernetes `Service`. Selectors can match [workload endpoints](../../reference/resources/workloadendpoint.mdx), + [host endpoints](../../reference/resources/hostendpoint.mdx) and ([namespaced](../../reference/resources/networkset.mdx) or + [global](../../reference/resources/globalnetworkset.mdx)) network sets. +- Source/destination IP address, protocol and port. + +If the rule contains multiple match criteria (for example, an IP and a port) then all match criteria must match +for the rule as a whole to match a packet. + +| Field | Description | Accepted Values | Schema | Default | +| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------- | ------------------------------------------- | ------- | +| nets | Match packets with IP in any of the listed CIDRs. | List of valid IPv4 CIDRs or list of valid IPv6 CIDRs (IPv4 and IPv6 CIDRs shouldn't be mixed in one rule) | list of cidrs | +| notNets | Negative match on CIDRs. Match packets with IP not in any of the listed CIDRs. | List of valid IPv4 CIDRs or list of valid IPv6 CIDRs (IPv4 and IPv6 CIDRs shouldn't be mixed in one rule) | list of cidrs | +| selector | Positive match on selected endpoints. If a `namespaceSelector` is also defined, the set of endpoints this applies to is limited to the endpoints in the selected namespaces. | Valid selector | [selector](#selector) | | +| notSelector | Negative match on selected endpoints. If a `namespaceSelector` is also defined, the set of endpoints this applies to is limited to the endpoints in the selected namespaces. | Valid selector | [selector](#selector) | | +| namespaceSelector | Positive match on selected namespaces. If specified, only workload endpoints in the selected Kubernetes namespaces are matched. Matches namespaces based on the labels that have been applied to the namespaces. Defines the scope that selectors will apply to, if not defined then selectors apply to the NetworkPolicy's namespace. Match a specific namespace by name using the `projectcalico.org/name` label. Select the non-namespaced resources like GlobalNetworkSet(s), host endpoints to which this policy applies by using `global()` selector. | Valid selector | [selector](#selector) | | +| ports | Positive match on the specified ports | | list of [ports](#ports) | | +| domains | Positive match on [domain names](#exact-and-wildcard-domain-names). | List of [exact or wildcard domain names](#exact-and-wildcard-domain-names) | list of strings | +| notPorts | Negative match on the specified ports | | list of [ports](#ports) | | +| serviceAccounts | Match endpoints running under service accounts. If a `namespaceSelector` is also defined, the set of service accounts this applies to is limited to the service accounts in the selected namespaces. | | [ServiceAccountMatch](#serviceaccountmatch) | | +| services | Match the specified service(s). If specified on egress rule destinations, no other selection criteria can be set. If specified on ingress rule sources, only positive or negative matches on ports can be specified. | | [ServiceMatch](#servicematch) | | + +:::note + +You cannot mix IPv4 and IPv6 CIDRs in a single rule using `nets` or `notNets`. If you need to match both, create 2 rules. + +::: + +When using selectors in network policy, remember that selectors only match (known) resources, but _rules_ match packets. +A rule with a selector `all()` won't match "all packets", it will match "packets from all in-scope endpoints and network sets". +To match all packets, do not include a selector in the rule at all. + +#### Empty match fields + +An empty value in a match field is treated as if the field were not set at all, not as an empty set that matches nothing. +For example, `nets: []` does not stop the rule from matching. +It leaves the rule unconstrained by IP address, so the rule matches any source or destination address. +The same is true of `notNets`, `ports`, and `notPorts`, and of an empty string in `selector`, `notSelector`, and `namespaceSelector`. +$[prodname] typically drops the empty value when it stores the resource, so the field is absent when you read the policy back. + +#### Understanding `notSelector` + +`notSelector` is somewhat subtle because the `not` in `notSelector` negates the packet match rather than the selector: + +- `selector: !has(foo)` matches packets from/to endpoints and network sets that do not have the label "foo". +- `notSelector: has(foo)` matches packets from/to **anywhere** (including outside the cluster), **except** traffic from/to endpoints and network sets that have the label "foo". + +#### Selector performance in EntityRules + +When rendering policy into the data plane, $[prodname] must identify the endpoints that match the selectors +in all active rules. This calculation is optimized for certain common selector types. +Using the optimized selector types reduces CPU usage (and policy rendering time) by orders of magnitude. +This becomes important at high scale (hundreds of active rules, hundreds of thousands of endpoints). + +The optimized operators are as follows: + +- `label == "value"` +- `label in { 'v1', 'v2' }` +- `has(label)` +- ` && ` is optimized if **either** `` or `` is + optimized. + +The following perform like `has(label)`. All endpoints with the label will be scanned to find matches: + +- `label contains 's'` +- `label starts with 's'` +- `label ends with 's'` + +The other operators, and in particular, `all()`, `!`, `||` and `!=` are not optimized. + +Examples: + +- `a == 'b'` - optimized +- `a == 'b' && has(c)` - optimized +- `a == 'b' || has(c)` - **not** optimized due to use of `||` +- `c != 'd'` - **not** optimized due to use of `!=` +- `!has(a)` - **not** optimized due to use of `!` +- `a == 'b' && c != 'd'` - optimized, `a =='b'` is optimized so `a == 'b' && ` is optimized. +- `c != 'd' && a == 'b'` - optimized, `a =='b'` is optimized so ` && a == 'b'` is optimized. + +### Exact and wildcard domain names + +The `domains` field is only valid for egress Allow rules. It restricts the +rule to apply only to traffic to one of the specified domains. If this field is specified, the +parent [Rule](#rule)'s `action` must be `Allow`, and `nets` and `selector` must both be left empty. + + + +:::note + +$[prodname] implements policy for domain names by learning the +corresponding IPs from DNS, then programming rules to allow those IPs. This means that +if multiple domain names A, B and C all map to the same IP, and there is domain-based +policy to allow A, traffic to B and C will be allowed as well. + +::: diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_httpmatch.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_httpmatch.mdx new file mode 100644 index 0000000000..4420e829e1 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_httpmatch.mdx @@ -0,0 +1,45 @@ +An HTTPMatch matches attributes of an HTTP request. The presence of an HTTPMatch clause on a Rule will cause that rule to only match HTTP traffic. Other application layer protocols will not match the rule. + +Example: + +```yaml +http: + methods: ['GET', 'PUT'] + paths: + - exact: '/projects/calico' + - prefix: '/users' + headers: + - header: 'x-forwarded-for' + operator: 'HasPrefix' + values: ['192.168.0.1', '192.168.0.254'] +``` + +| Field | Description | Schema | +| ------- | -------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------- | +| methods | Match HTTP methods. Case sensitive. [Standard HTTP method descriptions.](https://www.w3.org/Protocols/rfc2616/rfc2616-sec9.html) | list of strings | +| paths | Match HTTP paths. Case sensitive. | list of [HTTPPathMatch](#httppathmatch) | +| headers | Match HTTP headers. | list of [HTTPHeaderMatch](#httpheadermatch) | + +### HTTPPathMatch + +| Syntax | Example | Description | +| ------ | ------------------- | ------------------------------------------------------------------------------- | +| exact | `exact: "/foo/bar"` | Matches the exact path as written, not including the query string or fragments. | +| prefix | `prefix: "/keys"` | Matches any path that begins with the given prefix. | + +### HTTPHeaderMatch + +| Syntax | Example | Description | +| -------- | ---------------------------------- | ------------------------------------------------------------------------------------------ | +| header | `x-forwarded-for` | Name of a HTTP header. Header names are case insensitive, please use lowercase characters. | +| operator | `In` | Operator name to apply to the HTTP header value. Case sensitive. | +| values | `['192.168.0.1', '192.168.0.254']` | Values that the operator will test the HTTP header value against. Case sensitive. | + +The following operators are allowed: +- `Exists`: matches the HTTP request header if the specified header exists. `values` are ignored. +- `DoesNotExist`: matches the HTTP request header if the specified header does not exist. `values` are ignored. +- `HasPrefix`: matches the HTTP request header if the specified header has a prefix from any the values provided. +- `HasSuffix`: matches the HTTP request header if the specified header has a suffix from any the values provided. +- `In`: matches the HTTP request header if its value is in the set of values provided. +- `NotIn`: matches the HTTP request header if its value is not in the set of values provided. +- `MatchesRegex`: matches the HTTP request header if its value matches any of the regular expressions in values field. diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_icmp.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_icmp.mdx new file mode 100644 index 0000000000..1adb456472 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_icmp.mdx @@ -0,0 +1,4 @@ +| Field | Description | Accepted Values | Schema | Default | +| ----- | ------------------- | -------------------- | ------- | ------- | +| type | Match on ICMP type. | Can be integer 0-254 | integer | +| code | Match on ICMP code. | Can be integer 0-255 | integer | diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_ipnat.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_ipnat.mdx new file mode 100644 index 0000000000..9cfa2fb904 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_ipnat.mdx @@ -0,0 +1,6 @@ +IPNAT contains a single NAT mapping for a WorkloadEndpoint resource. + +| Field | Description | Accepted Values | Schema | Default | +| ---------- | ------------------------------------------- | ------------------ | ------ | ------- | +| internalIP | The internal IP address of the NAT mapping. | A valid IP address | string | | +| externalIP | The external IP address. | A valid IP address | string | | diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_license.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_license.mdx new file mode 100644 index 0000000000..9eff9db682 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_license.mdx @@ -0,0 +1,22 @@ +**How long does it take to get a new $[prodname] license?** + After you submit a sales purchase order to Tigera, 1-2 days. + +**Is there a grace period?** + No. + +**Does the web console display license expiration?** +Yes. The license indicator in the web console (top right banner) turns red when the license expires. + +![expiration](/img/calico-cloud/expiration.png) + +**What happens when a license expires or is invalid?** + Users can log in to the web console with read/access for all previously-created resources, but they cannot create any new $[prodname] resources. The web console may appear to function, but actions will not be applied; so it is important to proactively managed your license. + +**What happens if I add nodes beyond what I'm licensed for?** + +- Node limits are not currently enforced +- All $[prodname] features still work + +**How do I monitor my license?** + +- Use `kubectl` to get [license key information](../../reference/resources/licensekey.mdx#viewing-information-about-your-license-key) diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_non-cluster-binary-install.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_non-cluster-binary-install.mdx new file mode 100644 index 0000000000..ccb93567fc --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_non-cluster-binary-install.mdx @@ -0,0 +1,151 @@ +import NonClusterReadOnlyStep from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_non-cluster-read-only-step.mdx'; +import EnvironmentFile from '@site/calico-cloud_versioned_docs/version-24-1/_includes/components/EnvironmentFile'; + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + + +### Step 2: Download and extract the binary + +This step requires Docker, but it can be run from any machine with Docker installed. It doesn't have to be the host you will run it on (i.e your laptop is fine). + +1. Use the following command to download the $[nodecontainer] image. + + ```bash + docker pull $[registry]$[componentImage.cnxNode] + ``` + +1. Confirm that the image has loaded by typing `docker images`. + + ``` + REPOSITORY TAG IMAGE ID CREATED SIZE + $[registry]$[releases.0.components.cnx-node.image] $[releases.0.components.cnx-node.version] e07d59b0eb8a 2 minutes ago 42MB + ``` + +1. Create a temporary $[nodecontainer] container. + + ```bash + docker create --name container $[registry]$[componentImage.cnxNode] + ``` + +1. Copy the calico-node binary from the container to the local file system. + + ```bash + docker cp container:/bin/calico-node $[nodecontainer] + ``` + +1. Delete the temporary container. + + ```bash + docker rm container + ``` + +1. Set the extracted binary file to be executable. + + ```bash + chmod +x $[nodecontainer] + chown root:root $[nodecontainer] + ``` + +### Step 3: Copy the `calico-node` binary + +Copy the binary from Step 2 to the target machine, using any means (`scp`, `ftp`, USB stick, etc.). + +### Step 4: Create environment file + + + +### Step 5: Start Felix + +There are a few ways to start Felix: create a startup script, or manually configure Felix. + + + + +Felix should be started at boot by your init system and the init system +**must** be configured to restart Felix if it stops. Felix relies on +that behavior for certain configuration changes. + +If your distribution uses systemd, then you could use the following unit file: + +```bash +[Unit] +Description=Calico Felix agent +After=syslog.target network.target + +[Service] +User=root +EnvironmentFile=/etc/calico/calico.env +ExecStartPre=/usr/bin/mkdir -p /var/run/calico +ExecStart=/usr/local/bin/$[nodecontainer] -felix +KillMode=process +Restart=on-failure +LimitNOFILE=32000 + +[Install] +WantedBy=multi-user.target +``` + +Or, for upstart: + +```bash +description "Felix (Calico agent)" +author "Project Calico Maintainers " + +start on stopped rc RUNLEVEL=[2345] +stop on runlevel [!2345] + +limit nofile 32000 32000 + +respawn +respawn limit 5 10 + +chdir /var/run + +pre-start script + mkdir -p /var/run/calico + chown root:root /var/run/calico +end script + +exec /usr/local/bin/$[nodecontainer] -felix +``` + +**Start Felix** + +After you've configured Felix, start it via your init system. + +```bash +service calico-felix start +``` + + + + +Configure Felix by creating a file at `/kubernetes/calico/felix.cfg`. +{/*TODO-XREFS-CC +See [Felix configuration](/reference/felix/configuration) for help with environment variables. +*/} + +:::note + +Felix tries to detect whether IPv6 is available on your platform but +the detection can fail on older (or more unusual) systems. If Felix +exits soon after startup with `ipset` or `iptables` errors try +setting the `Ipv6Support` setting to `false`. + +::: + +Next, configure Felix to interact with a Kubernetes datastore. You +must set the `DatastoreType` setting to `kubernetes`. You must also set the environment variable `CALICO_KUBECONFIG` +to point to a valid kubeconfig for your kubernetes cluster and `CALICO_NETWORKING_BACKEND` to `none`. + +:::note + +For the Kubernetes datastore, Felix works in policy-only mode. Even though pod networking is +disabled on the baremetal host Felix is running on, policy can still be used to secure the host. + +::: + + + diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_non-cluster-read-only-step.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_non-cluster-read-only-step.mdx new file mode 100644 index 0000000000..84314c231f --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_non-cluster-read-only-step.mdx @@ -0,0 +1,21 @@ +import CreateKubeconfig from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_create-kubeconfig.mdx'; + +### Step 1: (Optional) Configure access for the non-cluster-host + +To run Calico Node as a container, it will need a kubeconfig. You can skip this step if you already have a kubeconfig ready to use. + + + +Run the following two commands to create a cluster role with read-only access and a corresponding cluster role binding. + +```bash +kubectl apply -f $[filesUrl_CE]/manifests/non-cluster-host-clusterrole.yaml +kubectl create clusterrolebinding $SA_NAME --serviceaccount=calico-system:$SA_NAME --clusterrole=non-cluster-host-read-only +``` + +:::note + +We include examples for systemd, but the commands can be +applied to other init daemons such as upstart. + +::: diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_ports.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_ports.mdx new file mode 100644 index 0000000000..4517f59290 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_ports.mdx @@ -0,0 +1,33 @@ +$[prodname] supports the following syntaxes for expressing ports. + +| Syntax | Example | Description | +| --------- | ---------- | ------------------------------------------------------------------- | +| int | 80 | The exact (numeric) port specified | +| start:end | 6040:6050 | All (numeric) ports within the range start ≤ x ≤ end | +| string | named-port | A named port, as defined in the ports list of one or more endpoints | + +An individual numeric port may be specified as a YAML/JSON integer. A port range or +named port must be represented as a string. For example, this would be a valid list of ports: + +```yaml +ports: [8080, '1234:5678', 'named-port'] +``` + +#### Named ports + +Using a named port in an `EntityRule`, instead of a numeric port, gives a layer of indirection, +allowing for the named port to map to different numeric values for each endpoint. + +For example, suppose you have multiple HTTP servers running as workloads; some exposing their HTTP +port on port 80 and others on port 8080. In each workload, you could create a named port called +`http-port` that maps to the correct local port. Then, in a rule, you could refer to the name +`http-port` instead of writing a different rule for each type of server. + +:::note + +Since each named port may refer to many endpoints (and $[prodname] has to expand a named port into +a set of endpoint/port combinations), using a named port is considerably more expensive in terms +of CPU than using a simple numeric port. We recommend that they are used sparingly, only where +the extra indirection is required. + +::: diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_rule.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_rule.mdx new file mode 100644 index 0000000000..60b88de64f --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_rule.mdx @@ -0,0 +1,46 @@ +A single rule matches a set of packets and applies some action to them. When multiple rules are specified, they +are executed in order. + +| Field | Description | Accepted Values | Schema | Default | +| ----------- | ------------------------------------------------------------------------------------------ | ------------------------------------------------------------ | ----------------------------- | ------- | +| metadata | Per-rule metadata. | | [RuleMetadata](#rulemetadata) | | +| action | Action to perform when matching this rule. | `Allow`, `Deny`, `Log`, `Pass` | string | | +| protocol | Positive protocol match. | `TCP`, `UDP`, `ICMP`, `ICMPv6`, `SCTP`, `UDPLite`, `1`-`255` | string \| integer | | +| notProtocol | Negative protocol match. | `TCP`, `UDP`, `ICMP`, `ICMPv6`, `SCTP`, `UDPLite`, `1`-`255` | string \| integer | | +| icmp | ICMP match criteria. | | [ICMP](#icmp) | | +| notICMP | Negative match on ICMP. | | [ICMP](#icmp) | | +| ipVersion | Positive IP version match. | `4`, `6` | integer | | +| source | Source match parameters. | | [EntityRule](#entityrule) | | +| destination | Destination match parameters. | | [EntityRule](#entityrule) | | +| http | Match HTTP request parameters. Application layer policy must be enabled to use this field. | | HTTPMatch | | + +After a `Log` action, processing continues with the next rule; `Allow` and `Deny` are immediate +and final and no further rules are processed. + +An `action` of `Pass` in a `NetworkPolicy` or `GlobalNetworkPolicy` will skip over the remaining policies and jump to the +first profile assigned to the endpoint, applying the policy configured in the +profile; if there are no Profiles configured for the endpoint the default applied action is `Deny`. + +### RuleMetadata + +Metadata associated with a specific rule (rather than the policy as a whole). The contents of the metadata does not affect how a rule is interpreted or enforced; it is +simply a way to store additional information for use by operators or applications that interact with $[prodname]. + +| Field | Description | Schema | Default | +| ----------- | ----------------------------------- | ----------------------- | ------- | +| annotations | Arbitrary non-identifying metadata. | map of string to string | | + +Example: + +```yaml +metadata: + annotations: + app: database + owner: devops +``` + +Annotations follow the +[same rules as Kubernetes for valid syntax and character set](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/#syntax-and-character-set). + +On Linux with the iptables data plane, rule annotations are rendered as comments in the form `-m comment --comment "="` on the iptables rule(s) that correspond +to the $[prodname] rule. diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_selector-scopes.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_selector-scopes.mdx new file mode 100644 index 0000000000..9d9fbc8c54 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_selector-scopes.mdx @@ -0,0 +1,20 @@ +Understanding scopes and the `all()` and `global()` operators: selectors have a scope of resources +that they are matched against, which depends on the context in which they are used. For example: + +- The `nodeSelector` in an `IPPool` selects over `Node` resources. + +- The top-level selector in a `NetworkPolicy` selects over the workloads _in the same namespace_ as the + `NetworkPolicy`. +- The top-level selector in a `GlobalNetworkPolicy` doesn't have the same restriction, it selects over all endpoints + including namespaced `WorkloadEndpoint`s and non-namespaced `HostEndpoint`s. + +- The `namespaceSelector` in a `NetworkPolicy` (or `GlobalNetworkPolicy`) _rule_ selects over the labels on namespaces + rather than workloads. + +- The `namespaceSelector` determines the scope of the accompanying `selector` in the entity rule. If no `namespaceSelector` + is present then the rule's `selector` matches the default scope for that type of policy. (This is the same namespace + for `NetworkPolicy` and all endpoints/network sets for `GlobalNetworkPolicy`) +- The `global()` operator can be used (only) in a `namespaceSelector` to change the scope of the main `selector` to + include non-namespaced resources such as [GlobalNetworkSet](../../reference/resources/globalnetworkset.mdx). + This allows namespaced `NetworkPolicy` resources to refer to global non-namespaced resources, which would otherwise + be impossible. diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_selectors.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_selectors.mdx new file mode 100644 index 0000000000..52baf960ae --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_selectors.mdx @@ -0,0 +1,50 @@ +A label selector is an expression which either matches or does not match a resource based on its labels. + +$[prodname] label selectors support a number of operators, which can be combined into larger expressions +using the boolean operators and parentheses. + +| Expression | Meaning | +| ------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| **Logical operators** | +| `( )` | Matches if and only if `` matches. (Parentheses are used for grouping expressions.) | +| `! ` | Matches if and only if `` does not match. **Tip:** `!` is a special character at the start of a YAML string, if you need to use `!` at the start of a YAML string, enclose the string in quotes. | +| ` && ` | "And": matches if and only if both ``, and, `` matches | +| \ || \ | "Or": matches if and only if either ``, or, `` matches. | +| **Match operators** | +| `all()` | Match all in-scope resources. To match _no_ resources, combine this operator with `!` to form `!all()`. | +| `global()` | Match all non-namespaced resources. Useful in a `namespaceSelector` to select global resources such as global network sets. | +| `k == 'v'` | Matches resources with the label 'k' and value 'v'. | +| `k != 'v'` | Matches resources without label 'k' or with label 'k' and value _not_ equal to `v` | +| `has(k)` | Matches resources with label 'k', independent of value. To match pods that do not have label `k`, combine this operator with `!` to form `!has(k)` | +| `k in { 'v1', 'v2' }` | Matches resources with label 'k' and value in the given set | +| `k not in { 'v1', 'v2' }` | Matches resources without label 'k' or with label 'k' and value _not_ in the given set | +| `k contains 's'` | Matches resources with label 'k' and value containing the substring 's' | +| `k starts with 's'` | Matches resources with label 'k' and value starting with the substring 's' | +| `k ends with 's'` | Matches resources with label 'k' and value ending with the substring 's' | + +Operators have the following precedence: + +- **Highest**: all the match operators +- Parentheses `( ... )` +- Negation with `!` +- Conjunction with `&&` +- **Lowest**: Disjunction with `||` + +For example, the expression + +``` +! has(my-label) || my-label starts with 'prod' && role in {'frontend','business'} +``` + +Would be "bracketed" like this: + +``` +((!(has(my-label)) || ((my-label starts with 'prod') && (role in {'frontend','business'})) +``` + +It would match: + +- Any resource that did not have label "my-label". +- Any resource that both: + - Has a value for `my-label` that starts with "prod", and, + - Has a role label with value either "frontend", or "business". diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_serviceaccountmatch.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_serviceaccountmatch.mdx new file mode 100644 index 0000000000..c3aff9c184 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_serviceaccountmatch.mdx @@ -0,0 +1,6 @@ +A ServiceAccountMatch matches service accounts in an EntityRule. + +| Field | Description | Schema | +| -------- | ------------------------------- | --------------------- | +| names | Match service accounts by name | list of strings | +| selector | Match service accounts by label | [selector](#selector) | diff --git a/calico-cloud_versioned_docs/version-24-1/_includes/content/_servicematch.mdx b/calico-cloud_versioned_docs/version-24-1/_includes/content/_servicematch.mdx new file mode 100644 index 0000000000..2d47fed02c --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/_includes/content/_servicematch.mdx @@ -0,0 +1,6 @@ +A ServiceMatch matches a service in an EntityRule. + +| Field | Description | Schema | +| --------- | ------------------------ | ------ | +| name | The service's name. | string | +| namespace | The service's namespace. | string | diff --git a/calico-cloud_versioned_docs/version-24-1/about/calico-product-editions.mdx b/calico-cloud_versioned_docs/version-24-1/about/calico-product-editions.mdx new file mode 100644 index 0000000000..d3507017e0 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/about/calico-product-editions.mdx @@ -0,0 +1,132 @@ +--- +description: Side-by-side comparison of Calico Cloud against Calico Open Source and Calico Enterprise across SaaS management, networking, security, and observability features. +--- + +import { CheckIcon } from '@chakra-ui/icons'; +import { CalicoProducts } from '/src/___new___/components'; + +# Calico product editions + + + +## Which product edition is right for me? + +| My needs | Calico product edition | +| ----- | ----- | +| I want open source, best-in-class networking, network security, and observability capabilities that can work across any Kubernetes distribution, for free. | Calico Open Source
[Get Started](/calico/latest/getting-started) | +| I’m a Calico Open Source user who wants to leverage some of the improved observability and policy management capabilities that are available in Calico Cloud, for free. | Calico Cloud Free Tier
[Sign up](https://auth.calicocloud.io/u/signup/identifier?state=hKFo2SBoMWV0M1RBUDlvWU83V2pGOS1ybVpCdmJGcDJRXzRSdaFur3VuaXZlcnNhbC1sb2dpbqN0aWTZIEJMclZBR2NmZzFacHRMclRIOFBzVEdNdVozOFlOSkpio2NpZNkgcTZ4MVM2dHdxZ0pHOVZLSEQzRGR0S0tEQTB2MlNuYkU) | +| My organization wants a **fully managed SaaS** platform for network security and observability. | Calico Cloud
[Get Started](../get-started/index.mdx) | +| My organization wants a **self-managed platform** for network security and observability. | Calico Enterprise
[Get Started](/calico-enterprise/latest/getting-started/) | + +## Feature comparison matrix + +| | Calico Open Source | Calico Cloud Free Tier\* | Calico Cloud | Calico Enterprise | +| ----- | :-----: | :-----: | :-----: | :-----: | +| **Management and Support** | | | | | +| Mutli-cluster security controls management | | | | | +| Data retention | In-memory | 24 hours | 7 days | Unlimited | +| Number of clusters | Unlimited | One | Unlimited | Unlimited | +| Number of users | N/A | One | Unlimited | Unlimited | +| Support and maintenance | Community-driven | Community-driven | Standard/Business | Standard/Business | +| **Networking** | | | | | +| High performance, scalable pod networking | | | | | +| Advanced IP address management | | | | | +| Direct infrastructure peering without the overlay | | | | | +| eBPF data plane | | | | | +| Windows data plane | | | | | +| nftables data plane | | | | | +| iptables data plane | | | | | +| VPP data plane | | | | | +| Multiple Calico networks on a pod | | | | | +| Dual ToR peering | | | | | +| Ingress gateway | | | | | +| Egress gateway | | | | | +| Cluster mesh | | | | | +| **Network Security** | | | | | +| Seamless support for Kubernetes network policy | | | | | +| Label-based policies for K8s and non-K8s workloads | | | | | +| Namespace and cluster-wide scope | | | | | +| Global default deny policy design | | | | | +| Application layer policy | | | | | +| Policy for services | | | | | +| Policy board | | View only | | | +| DNS/FQDN-based policy | | | | | +| Hierarchical tiered network policy | | | | | +| Policy recommendations | | Manual workflow | | | +| Staged network policy | | | | | +| Preview staged policies | | | | | +| Network sets to limit IP ranges for egress and ingress traffic to workloads | | | | | +| Data-in-transit encryption | | | | | +| Universal firewall integration | | | | | +| Workload-based IDS/IPS | | | | | +| Deep packet inspection | | | | | +| DDoS protection | | | | | +| Workload-centric WAF | | | | | +| SIEM integrations | | | | | +| **Network Security for VMs and Bare Metal** | | | | | +| Restrict traffic to/from hosts and VMs using network policy | | | | | +| Automatic host endpoints | | | | | +| Apply policy to host-forwarded traffic | | | | | +| **Observability** | | | | | +| Flow logs API | | | | | +| Calico Whisker web console | | | | | +| Dynamic service and threat graph | | | | | +| Application level observability | | | | | +| Dynamic packet capture | | | | | +| Flow visualizer | | | | | +| Logs (flow) | | | | | +| Logs (http traffic, audit, bgp, dns, events) | | | | | +| Dashboards | | \*\* | | | +| Alerts | | | | | + +\* Calico Cloud Free Tier requires a cluster with Calico Open Source 3.30 or higher. + +\*\* Calico Cloud Free Tier includes some of the dashboards that are available in Calico Cloud and Calico Enterprise. + +## How to get started with Calico + +Calico powers 100M+ containers across 8M+ nodes in 166 countries, and is supported across all major cloud providers and Kubernetes distributions. + +### Ready to get started? + +Start a free trial or request a demo to see Calico in action. + +* [Sign up for a Calico Cloud Free Tier account](https://www.calicocloud.io/?code=free) +* [Schedule a 30-minute demo with our experts](https://www.tigera.io/demo/) + +### Installation guides + +* [Install Calico Open Source](/calico/latest/getting-started) +* [Install Calico Cloud Free Tier](../free/connect-cluster-free.mdx) +* [Install Calico Cloud](../get-started/install-cluster.mdx) +* [Install Calico Enterprise](/calico-enterprise/latest/getting-started/) + +## How to engage + +### Learning resources + +* [Blog](https://www.tigera.io/blog/) +* [Certifications](https://www.tigera.io/lp/calico-certification/) (self-paced) +* [Product tutorials](https://www.tigera.io/tutorials/) (self-paced) +* [Learn guides](https://www.tigera.io/learn/guides/kubernetes-networking/) +* [Webinars and workshops](https://www.tigera.io/events/) (live and on demand) +* [Resources](https://www.tigera.io/resources/) + +### Get involved + +* [Calico Open Source Community](https://www.tigera.io/project-calico/community/) +* GitHub + * [Project Calico](https://github.com/projectcalico/calico) + * [Tigera](https://github.com/tigera) + +### Get in touch + +* [Slack](https://calicousers.slack.com/) (Calico Open Source users) +* [YouTube](https://www.youtube.com/channel/UCFpTnXDNcBoXI4gqCDmegFA) (@ProjectCalico) +* LinkedIn + * [Tigera](https://www.linkedin.com/company/tigera/) + * [Project Calico](https://www.linkedin.com/company/project-calico/) +* X + * [Tigera](https://x.com/tigeraio) + * [Project Calico](https://x.com/projectcalico) +* [Contact us](https://www.tigera.io/contact/) \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/about/index.mdx b/calico-cloud_versioned_docs/version-24-1/about/index.mdx new file mode 100644 index 0000000000..5e21e8f4c6 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/about/index.mdx @@ -0,0 +1,260 @@ +--- +description: Overview of Calico Cloud, the Tigera-hosted managed SaaS for Kubernetes networking, network security, and observability across connected clusters. +--- + +import { DocCardLink, DocCardLinkLayout, PaidProductDocCardLink } from '/src/___new___/components'; +import { CheckIcon } from '@chakra-ui/icons'; +import { CalicoProducts } from '/src/___new___/components'; + +# About Calico + +## What is Calico? + +Calico is a single platform for networking, network security, and observability for any Kubernetes distribution in the cloud, on-premises, or at the edge. +Whether you're just starting with Kubernetes or operating at scale, Calico's open source, enterprise, and cloud editions provide the networking, security, and observability you need. + +The key advantages of Calico are: + +* A single platform to address all networking, network security, and observability needs for Kubernetes environments +* Consistent networking and network security controls for any Kubernetes distribution, ensuring workload portability +* Ability to scale networking and network security to multi-cluster applications, VMs, and bare metal servers without additional software + +The Calico portfolio of products includes Calico Open Source, Calico Enterprise (self-managed), and Calico Cloud (fully-managed SaaS). +Calico Cloud Free Tier is a free version of Calico Cloud that focuses on observability and policy management for a single cluster. +All of this is built on Calico Open Source, the most widely used container networking and security solution. + +## Calico overview + +### Networking + + + + + + {/* We don't have a good docs page for this one. + */} + + +### Network security + + +{/* */} + + + + + + +### Threat detection, observability, and incident response + + + + + + + + +### CI/CD automation and tools + + + {/**/} + + + + + + +## Calico product editions + + + +## Which product edition is right for me? + +| My needs | Calico product edition | +| ----- | ----- | +| I want open source, best-in-class networking, network security, and observability capabilities that can work across any Kubernetes distribution, for free. | Calico Open Source
[Get Started](/calico/latest/getting-started) | +| I’m a Calico Open Source user who wants to leverage some of the improved observability and policy management capabilities that are available in Calico Cloud, for free. | Calico Cloud Free Tier
[Sign up](https://auth.calicocloud.io/u/signup/identifier?state=hKFo2SBoMWV0M1RBUDlvWU83V2pGOS1ybVpCdmJGcDJRXzRSdaFur3VuaXZlcnNhbC1sb2dpbqN0aWTZIEJMclZBR2NmZzFacHRMclRIOFBzVEdNdVozOFlOSkpio2NpZNkgcTZ4MVM2dHdxZ0pHOVZLSEQzRGR0S0tEQTB2MlNuYkU) | +| My organization wants a **fully managed SaaS** platform for network security and observability. | Calico Cloud
[Get Started](../get-started/index.mdx) | +| My organization wants a **self-managed platform** for network security and observability. | Calico Enterprise
[Get Started](/calico-enterprise/latest/getting-started/) | + +## Feature comparison matrix + +| | Calico Open Source | Calico Cloud Free Tier\* | Calico Cloud | Calico Enterprise | +| ----- | :-----: | :-----: | :-----: | :-----: | +| **Management and Support** | | | | | +| Mutli-cluster security controls management | | | | | +| Data retention | In-memory | 24 hours | 7 days | Unlimited | +| Number of clusters | Unlimited | One | Unlimited | Unlimited | +| Number of users | N/A | One | Unlimited | Unlimited | +| Support and maintenance | Community-driven | Community-driven | Standard/Business | Standard/Business | +| **Networking** | | | | | +| High performance, scalable pod networking | | | | | +| Advanced IP address management | | | | | +| Direct infrastructure peering without the overlay | | | | | +| eBPF data plane | | | | | +| Windows data plane | | | | | +| nftables data plane | | | | | +| iptables data plane | | | | | +| VPP data plane | | | | | +| Multiple Calico networks on a pod | | | | | +| Dual ToR peering | | | | | +| Ingress gateway | | | | | +| Egress gateway | | | | | +| Cluster mesh | | | | | +| **Network Security** | | | | | +| Seamless support for Kubernetes network policy | | | | | +| Label-based policies for K8s and non-K8s workloads | | | | | +| Namespace and cluster-wide scope | | | | | +| Global default deny policy design | | | | | +| Application layer policy | | | | | +| Policy for services | | | | | +| Policy board | | View only | | | +| DNS/FQDN-based policy | | | | | +| Hierarchical tiered network policy | | | | | +| Policy recommendations | | Manual workflow | | | +| Staged network policy | | | | | +| Preview staged policies | | | | | +| Network sets to limit IP ranges for egress and ingress traffic to workloads | | | | | +| Data-in-transit encryption | | | | | +| Universal firewall integration | | | | | +| Workload-based IDS/IPS | | | | | +| Deep packet inspection | | | | | +| DDoS protection | | | | | +| Workload-centric WAF | | | | | +| SIEM integrations | | | | | +| **Network Security for VMs and Bare Metal** | | | | | +| Restrict traffic to/from hosts and VMs using network policy | | | | | +| Automatic host endpoints | | | | | +| Apply policy to host-forwarded traffic | | | | | +| **Observability** | | | | | +| Flow logs API | | | | | +| Calico Whisker web console | | | | | +| Dynamic service and threat graph | | | | | +| Application level observability | | | | | +| Dynamic packet capture | | | | | +| Flow visualizer | | | | | +| Logs (flow) | | | | | +| Logs (http traffic, audit, bgp, dns, events) | | | | | +| Dashboards | | \*\* | | | +| Alerts | | | | | + +\* Calico Cloud Free Tier requires a cluster with Calico Open Source 3.30 or higher. + +\*\* Calico Cloud Free Tier includes some of the dashboards that are available in Calico Cloud and Calico Enterprise. + +## How to get started with Calico + +Calico powers 100M+ containers across 8M+ nodes in 166 countries, and is supported across all major cloud providers and Kubernetes distributions. + +### Ready to get started? + +Start a free trial or request a demo to see Calico in action. + +* [Sign up for a Calico Cloud Free Tier account](https://www.calicocloud.io/?code=free) +* [Schedule a 30-minute demo with our experts](https://www.tigera.io/demo/) + +### Installation guides + +* [Install Calico Open Source](/calico/latest/getting-started) +* [Install Calico Cloud Free Tier](../free/connect-cluster-free.mdx) +* [Install Calico Cloud](../get-started/install-cluster.mdx) +* [Install Calico Enterprise](/calico-enterprise/latest/getting-started/) + +## How to engage + +### Learning resources + +* [Blog](https://www.tigera.io/blog/) +* [Certifications](https://www.tigera.io/lp/calico-certification/) (self-paced) +* [Product tutorials](https://www.tigera.io/tutorials/) (self-paced) +* [Learn guides](https://www.tigera.io/learn/guides/kubernetes-networking/) +* [Webinars and workshops](https://www.tigera.io/events/) (live and on demand) +* [Resources](https://www.tigera.io/resources/) + +### Get involved + +* [Calico Open Source Community](https://www.tigera.io/project-calico/community/) +* GitHub + * [Project Calico](https://github.com/projectcalico/calico) + * [Tigera](https://github.com/tigera) + +### Get in touch + +* [Slack](https://calicousers.slack.com/) (Calico Open Source users) +* [YouTube](https://www.youtube.com/channel/UCFpTnXDNcBoXI4gqCDmegFA) (@ProjectCalico) +* LinkedIn + * [Tigera](https://www.linkedin.com/company/tigera/) + * [Project Calico](https://www.linkedin.com/company/project-calico/) +* X + * [Tigera](https://x.com/tigeraio) + * [Project Calico](https://x.com/projectcalico) +* [Contact us](https://www.tigera.io/contact/) \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/compliance/configure-http-proxy.mdx b/calico-cloud_versioned_docs/version-24-1/compliance/configure-http-proxy.mdx new file mode 100644 index 0000000000..59883c8b69 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/compliance/configure-http-proxy.mdx @@ -0,0 +1,99 @@ +--- +description: Route outbound container traffic through an HTTP proxy on clusters connected to Calico Cloud by configuring the Installation custom resource via the Tigera Operator. +--- + +# Configure an outbound HTTP proxy + +$[prodname] supports the use of HTTP proxies for connections that leave the cluster and originate from $[prodname] containers. + +`HTTP_PROXY`, `HTTPS_PROXY`, and `NO_PROXY` environment variables can be set on $[prodname] containers by using a mutating admission webhook, or by configuration through the Installation CR if preferred. In the latter case, the Tigera Operator will be responsible for setting proxy environment variables on $[prodname] containers that egress out of the cluster. + +With either environment variable configuration method, the Tigera Operator will detect the proxy environment variables that are set on $[prodname] containers that egress out of the cluster and ensure that network policy in the calico-system tier allows these connections. + +This page will outline how to configure an HTTP proxy using the Installation CR. + +:::caution + +$[prodname] assumes that you have exempted in-cluster destinations (including the Kubernetes API server) from proxying via the `NO_PROXY` variable. + +::: + +## Configuration via Helm +If you wish to manually configure $[prodname] via Helm to set proxy environment variables on $[prodname] containers (rather than using a mutating admission webhook), you will first need to familiarize yourself with how to [customize the $[prodname] Helm chart using values.yaml](../get-started/install-automated#prepare-your-valuesyaml-with-customizations). + +Following the instructions linked above, retrieve your `.yaml`. Edit the file to set the `installer.components.installation.proxy` path, which configures the Installation CR with your proxy. This instructs the Tigera Operator to program your proxy on relevant containers that it deploys. e.g.: + +```yaml +installer: + components: + installation: + proxy: + httpProxy: + httpsProxy: + noProxy: +``` + +In your `.yaml` file, you'll also need to set the `installer.components.ccInstaller.operatorDeployment.spec.template.spec.containers` path, which programs your proxy on the calico-cloud-controller-manager pod. + +```yaml +installer: + components: + ccInstaller: + operatorDeployment: + spec: + template: + spec: + containers: + - name: calico-cloud + env: + - name: HTTP_PROXY + value: + - name: HTTPS_PROXY + value: + - name: NO_PROXY + value: + +``` + +Once you've completed your edits to your `.yaml`, you can [install](../get-started/install-automated#install-calico-cloud-as-part-of-an-automated-workflow-1) $[prodname] using your customized values. + +:::caution + +Ensure that the other required values for the Helm chart are set in the values.yaml file, as indicated by the generated Helm install command. + +::: + +## Configuration via Manifest +If you wish to manually configure $[prodname] via manifests to set proxy environment variables on $[prodname] containers (rather than using a mutating admission webhook), you’ll need to edit the manifests generated at install and reinstall time. + +First, you’ll need to edit the cc-operator deploy.yaml so that the `calico-cloud/calico-cloud-controller-manager` Deployment resource has your required environment variables (`HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`) set in its `env` section. + +Second, you’ll need to edit the managed-cluster deploy.yaml so that the `calico-cloud/default` Installer resource has the Installation CR proxy configuration set. This is set at the `.spec.components.installation.proxy` path, e.g. + +``` +apiVersion: operator.calicocloud.io/v1 +kind: Installer +metadata: + name: default + namespace: calico-cloud +spec: + clusterName: "proxy-cluster" + resourceVersion: "20241209202443" + components: + installation: + proxy: + httpProxy: + httpsProxy: + noProxy: +``` + +## Validating Configuration +Once your install or reinstall has completed, you can: +* Check the Installation CR to validate that the `.spec.proxy` path has been set as expected +* Check the calico-cloud-controller-manager Deployment to validate that the environment variables have been set on the container as expected + +:::caution + +You can manually edit the two places above to configure $[prodname] to use a proxy, but keep in mind that the reinstall process (via Helm or manifest) may overwrite your changes if you did not follow the steps in previous sections. + +::: diff --git a/calico-cloud_versioned_docs/version-24-1/compliance/encrypt-cluster-pod-traffic.mdx b/calico-cloud_versioned_docs/version-24-1/compliance/encrypt-cluster-pod-traffic.mdx new file mode 100644 index 0000000000..507bfd43eb --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/compliance/encrypt-cluster-pod-traffic.mdx @@ -0,0 +1,233 @@ +--- +description: Turn on WireGuard for clusters connected to Calico Cloud to encrypt inter-node pod and host-network traffic, with IPv4 and IPv6 support via FelixConfiguration. +--- + +# Encrypt data in transit + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## Big picture + +Enable WireGuard to secure on the wire in-cluster pod traffic in a $[prodname] cluster. + +## Value + +When this feature is enabled, $[prodname] automatically creates and manages WireGuard tunnels between nodes providing transport-level security for inter-node, in-cluster pod traffic. WireGuard provides [formally verified](https://www.wireguard.com/formal-verification/) secure and [performant tunnels](https://www.wireguard.com/performance/) without any specialized hardware. For a deep dive in to WireGuard implementation, see this [white paper](https://www.wireguard.com/papers/wireguard.pdf). + +$[prodname] supports WireGuard encryption for both IPv4 and IPv6 traffic. These can be independently enabled in the FelixConfiguration resource: `wireguardEnabled` +enables encrypting IPv4 traffic over an IPv4 underlay network and `wireguardEnabledV6` enables encrypting IPv6 traffic over an IPv6 underlay network. + +## Before you begin + +**Terminology** + + - Inter-node pod traffic: Traffic leaving a pod from one node destined to a pod on another node + - Inter-node, host-network traffic: traffic generated by the node itself or a host-networked-pod destined to another node or host-networked-pod + - Same-node pod traffic: Traffic between pods on the same node + +**Supported encryption** + +- Inter-node pod traffic: IPv4 only +- Inter-node, host-network traffic, IPv4/IPv6: supported only on managed clusters deployed on EKS and AKS + +**Unsupported** + +- Encrypted same-node pod traffic +- GKE +- Using your own custom keys to encrypt traffic + +**Required** + +- On all nodes in the cluster that you want to participate in $[prodname] encryption, verify that the operating system(s) on the nodes are [installed with WireGuard](https://www.wireguard.com/install/). + + :::note + + Some node operating systems do not support WireGuard, or do not have it installed by default. Enabling $[prodname] WireGuard encryption does not require all nodes to be installed with WireGuard. However, traffic to or from a node that does not have WireGuard installed, will not be encrypted. + + ::: + +- IP addresses for every node in the cluster. This is required to establish secure tunnels between the nodes. $[prodname] can automatically do this using [IP autodetection methods](../networking/ipam/ip-autodetection.mdx). + +## How to + +- [Install WireGuard](#install-wireguard) +- [Enable WireGuard for a cluster](#enable-wireguard-for-a-cluster) +- [Verify encryption is enabled](#verify-encryption-is-enabled) +- [Disable WireGuard for an individual node](#disable-wireguard-for-an-individual-node) +- [Disable WireGuard for a cluster](#disable-wireguard-for-a-cluster) + +### Install WireGuard + +WireGuard is included in Linux 5.6+ kernels, and has been backported to earlier Linux kernels in some Linux distributions. + +Install WireGuard on cluster nodes using [instructions for your operating system](https://www.wireguard.com/install/). Note that you may need to reboot your nodes after installing WireGuard to make the kernel modules available on your system. + +Use the following instructions for these platforms that are not listed on the WireGuard installation page, before proceeding to [enabling WireGuard](#enable-wireguard-for-a-cluster). + + + + +To install WireGuard on the default Amazon Machine Image (AMI): + +```bash +sudo yum install kernel-devel-`uname -r` -y +sudo yum install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm -y +sudo curl -o /etc/yum.repos.d/jdoss-wireguard-epel-7.repo https://copr.fedorainfracloud.org/coprs/jdoss/wireguard/repo/epel-7/jdoss-wireguard-epel-7.repo +sudo yum install wireguard-dkms wireguard-tools -y +``` + + + + +AKS cluster nodes run Ubuntu with a kernel that has WireGuard installed already, so there is no manual installation required. + + + + +To install WireGuard for OpenShift v4.8: + +1. Install requirements: + + - [CoreOS Butane](https://coreos.github.io/butane/getting-started/) + - [OpenShift CLI](https://docs.openshift.com/container-platform/4.2/cli_reference/openshift_cli/getting-started-cli.html) + +1. Download and configure the tools needed for kmods. + +```bash +FAKEROOT=$(mktemp -d) +git clone https://github.com/tigera/kmods-via-containers +cd kmods-via-containers +make install FAKEROOT=${FAKEROOT} +cd .. +git clone https://github.com/tigera/kvc-wireguard-kmod +cd kvc-wireguard-kmod +make install FAKEROOT=${FAKEROOT} +cd .. +``` + +1. Configure/edit `${FAKEROOT}/root/etc/kvc/wireguard-kmod.conf`. + + a. You must then set the URLs for the `KERNEL_CORE_RPM`, `KERNEL_DEVEL_RPM` and `KERNEL_MODULES_RPM` packages in the conf file `$FAKEROOT/etc/kvc/wireguard-kmod.conf`. Obtain copies for `kernel-core`, `kernel-devel`, and `kernel-modules` rpms from [RedHat Access](https://access.redhat.com/downloads/content/package-browser) and host it in an http file server that is reachable by your OCP workers. + + b. For help configuring `kvc-wireguard-kmod/wireguard-kmod.conf` and WireGuard version to kernel version compatibility, see the [kvc-wireguard-kmod README file](https://github.com/tigera/kvc-wireguard-kmod#quick-config-variables-guide). + +1. Get RHEL Entitlement data from your own RHEL8 system from a host in your cluster. + + ```bash + tar -czf subs.tar.gz /etc/pki/entitlement/ /etc/rhsm/ /etc/yum.repos.d/redhat.repo + ``` + +1. Copy the `subs.tar.gz` file to your workspace and then extract the contents using the following command. + + ```bash + tar -x -C ${FAKEROOT}/root -f subs.tar.gz + ``` + +1. Transpile your machine config using [CoreOS Butane](https://coreos.github.io/butane/getting-started/). + + ```bash + cd kvc-wireguard-kmod + make ignition FAKEROOT=${FAKEROOT} > mc-wg.yaml + ``` + +1. With the KUBECONFIG set for your cluster, run the following command to apply the MachineConfig which will install WireGuard across your cluster. + ```bash + oc create -f mc-wg.yaml + ``` + + + + +### Enable WireGuard for a cluster + +Enable IPv4 WireGuard encryption across all the nodes using the following command. + +```bash +kubectl patch felixconfiguration default --type='merge' -p '{"spec":{"wireguardEnabled":true}}' +``` + +Enable IPv6 WireGuard encryption across all the nodes using the following command. + +```bash +kubectl patch felixconfiguration default --type='merge' -p '{"spec":{"wireguardEnabledV6":true}}' +``` + +To enable both IPv4 and IPv6 WireGuard encryption across all the nodes, use the following command. + +```bash +kubectl patch felixconfiguration default --type='merge' -p '{"spec":{"wireguardEnabled":true,"wireguardEnabledV6":true}}' +``` + +{/*TODO-CC-XREFS For OpenShift, add the Felix configuration with WireGuard enabled [under custom resources](/getting-started/install-on-clusters/openshift/installation/#provide-additional-configuration).*/} + +:::note + +The above command can be used to change other WireGuard attributes. For a list of other WireGuard parameters and configuration evaluation, see the [Felix configuration](../reference/resources/felixconfig.mdx#felix-configuration-definition). + +::: + +We recommend that you review and modify the MTU used by $[prodname] networking when WireGuard is enabled to increase network performance. Follow the instructions in the [Configure MTU to maximize network performance](../networking/configuring/mtu.mdx) guide to set the MTU to a value appropriate for your network. + +### Verify encryption is enabled + +To verify that the nodes are configured for WireGuard encryption, check the node status set by Felix using `kubectl`. For example: + +```bash +kubectl get node -o yaml +... +kind: Node +metadata: + annotations: + projectcalico.org/WireguardPublicKey: jlkVyQYooZYzI2wFfNhSZez5eWh44yfq1wKVjLvSXgY= +... +``` + +### Enable WireGuard statistics + +Since v3.11.1, WireGuard statistics are now automatically enabled with the enable WireGuard setting(s) mentioned above. + +### View WireGuard statistics + +To view WireGuard statistics in the web console, you must enable them. From the left navbar, click **Dashboard**, and the Layout Settings icon. + +![WireGuard Dashboard Toggle](/img/calico-enterprise/wireguard/stats-toggle.png) + +### Disable WireGuard for an individual node + +To disable WireGuard on a specific node with WireGuard installed, modify the node-specific Felix configuration. e.g., to turn off encryption for pod traffic on node `my-node`, use the following command. This command disables WireGuard for both IPv4 and IPv6, modify it accordingly if disabling only either IP version: + +```bash +cat < + + + + + diff --git a/calico-cloud_versioned_docs/version-24-1/compliance/istio/about-istio-ambient.mdx b/calico-cloud_versioned_docs/version-24-1/compliance/istio/about-istio-ambient.mdx new file mode 100644 index 0000000000..b7fc6197fa --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/compliance/istio/about-istio-ambient.mdx @@ -0,0 +1,53 @@ +--- +description: Overview of the bundled Istio Ambient Mode service mesh for clusters connected to Calico Cloud, with node-level zTunnel proxies and mTLS managed by the Tigera Operator. +--- + +# Istio Ambient Mode + +You can use $[prodname] to deploy and manage an Istio service mesh on your cluster. +$[prodname] installs Istio in ambient mode, which conserves resources while providing the same robust mTLS encryption for your workloads. + +:::note + +Istio Ambient Mode is a tech preview feature. +Tech preview features are subject to significant changes before they become GA. + +::: + +## About Istio Ambient Mode + +Istio is a service mesh that manages and secures communication between microservices. +Typically, Istio uses sidecar proxies that are deployed alongside every pod in the service mesh. +At scale, running these sidecar proxies can be difficult to manage and a drain on resources. + +Istio Ambient Mode is a simplified service mesh architecture that removes the need for a sidecar proxy next to every pod. +Instead, it uses node-level components for shared security and a layered approach for advanced traffic management. +This design saves on computing resources and simplifies operations. + +## About Istio Ambient Mode on Calico + +$[prodname] provides a bundled version of Istio that can be installed and managed by the Tigera Operator. + +This integration automates the lifecycle of the Istio components to reduce manual configuration overhead. +CVEs are addressed as part of the regular $[prodname] patch release cadence. +Administrators provision the Istio service mesh by defining a standard `Istio` custom resource. + +### The enhanced zTunnel proxy + +The zTunnel component in Istio Ambient Mode is a lightweight proxy that runs on every node. + +Its main job is to handle encryption, authentication, and policy enforcement for traffic at Layer 4. + +A challenge in the original Istio Ambient Mode is that when traffic is routed through the zTunnel, it gets placed into a tunnel on a specific port (15008). +This change makes it impossible for existing Layer 3 or Layer 4 network policies (like those from Calico) to see the original destination port of the traffic. + +Calico addresses this by using an enhanced zTunnel that is modified to preserve the original destination port. +This modification allows existing Calico and Kubernetes network policies to continue functioning exactly as they did before, without needing any rewrites, even though the traffic is now encrypted with mTLS. + +These zTunnel enhancements are not compatible with Istio's application-layer Waypoint proxy. +If you deploy Waypoint, the reported destination ports will follow the original behavior. +Existing network policies need to be adapted to allow communication to port 15008. + +## Additional resources +* [Overview of Istio ambient mode](https://istio.io/latest/docs/ambient/overview/). +* [Ambient and Kubernetes NetworkPolicy](https://istio.io/latest/docs/ambient/usage/networkpolicy/) diff --git a/calico-cloud_versioned_docs/version-24-1/compliance/istio/deploy-istio-ambient.mdx b/calico-cloud_versioned_docs/version-24-1/compliance/istio/deploy-istio-ambient.mdx new file mode 100644 index 0000000000..ec104f45a5 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/compliance/istio/deploy-istio-ambient.mdx @@ -0,0 +1,161 @@ +--- +description: Deploy the bundled Istio Ambient Mode service mesh on clusters connected to Calico Cloud by applying the Istio custom resource, with prerequisites and limitations. +--- + +# Deploy Istio Ambient Mode on your cluster + +You can deploy Calico's bundled version of Istio in ambient mode to provide mTLS encryption to your workloads. + +:::note + +Istio Ambient Mode is a tech preview feature. +Tech preview features are subject to significant changes before they become GA. + +::: + +## Limitations + +* [Application layer network policies](../../network-policy/application-layer-policies/alp.mdx) are not compatible with the Istio service mesh. +* Istio Ambient Mode does not work together with [workload-based web application firewalls](../../threat/web-application-firewall.mdx). +* The service mesh is not supported for use on clusters that are also part of a [cluster mesh](../../multicluster/index.mdx). +* Destination ports are preserved only when Istio is deployed without Waypoint. + If you deploy Waypoint, all traffic through Waypoint will show port 15008 as its destination port. +* Connect-time load balancing is not compatible with Istio Ambient Mode. + +## Prerequisites + +* $[prodname] is installed and managed by the Tigera Operator. + +## Install Istio in ambient mode on your cluster + +You can create an Istio service mesh in ambient mode by creating the `Istio` custom resource. + +* To install Istio in ambient mode, apply the `Istio` custom resource to your cluster: + + ```bash + cat < istio.io/dataplane-mode=ambient + ``` + Replace `` with the namespace you want to include in the mesh. + + ```bash title='Adding a pod to the Istio service mesh' + kubectl label pod --namespace= istio.io/dataplane-mode=ambient + ``` + Replace the following: + * ``: The name of the pod you want to include in the mesh. + * ``: The namespace your pod is in. + +## Removing Istio + +If you want to remove Istio, first remove the labels you applied to pods and namespaces. +When that's done, you can delete the `Istio` custom resource. + +1. Remove the label from namespaces and pods by running the following commands: + + ```bash + kubectl label namespaces --all istio.io/dataplane-mode=ambient- + kubectl label pods --all --all-namespaces istio.io/dataplane-mode=ambient- + ``` +1. Remove the `Istio` custom resource: + + ```bash + kubectl delete istio.operator.tigera.io default + ``` + +## Troubleshooting commands + +Check whether Istio pods are deployed: + +```bash +kubectl get pods -n calico-system | grep 'istio\|ztunnel' +``` + +Check whether Istio CRDs are deployed: + +```bash +kubectl get crd | grep istio +``` + +Check which pods and namespaces are in the mesh: + +* Requires [istioctl](https://istio.io/latest/docs/ops/diagnostic-tools/istioctl/). + +```bash +istioctl ztunnel-config workloads -n calico-system +``` + +Check for errors logged by the zTunnel component: + +```bash +ZTUNNEL_PODS=$(kubectl get pod -n calico-system \ + -l app.kubernetes.io/name=ztunnel \ + -o jsonpath='{.items[*].metadata.name}') + +for P in $ZTUNNEL_PODS; do + echo "--- Checking logs for pod: $P ---" + kubectl logs $P -n calico-system 2>/dev/null | \ + grep -i error | \ + grep -i app1 +done +``` + +## Additional resources + +* [Overview of Istio ambient mode](https://istio.io/latest/docs/ambient/overview/). +* [Configuration options](../../reference/installation/api). \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/free/connect-cluster-free.mdx b/calico-cloud_versioned_docs/version-24-1/free/connect-cluster-free.mdx new file mode 100644 index 0000000000..60715baa77 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/free/connect-cluster-free.mdx @@ -0,0 +1,184 @@ +--- +description: Connect a Kubernetes cluster to Calico Cloud Free Tier so it reports network observability data to a Tigera-managed dashboard. +title: Connect a cluster +--- + +# Connect a cluster to Calico Cloud Free Tier + +This page shows you how to connect your Kubernetes cluster to Calico Cloud Free Tier. + +:::tip +Don't have a cluster ready to connect? +Try the [Calico Cloud Free Tier quickstart guide](quickstart.mdx) to create a cluster locally and connect it to Calico Cloud Free Tier. +::: + +## Prerequisites + +* You have a Kubernetes cluster. + * Your `kubectl` tool is configured to communicate with your cluster. + * If you're working in a restricted network environment, you are able to create allow rules for TCP egress traffic to the Calico Cloud management plane. +* You [installed Calico Open Source 3.30 or later on your cluster](/calico/latest/getting-started). + * If you upgraded from an earlier version of Calico Open Source, you [enabled the `Goldmane` custom resource](/calico/latest/observability/enable-whisker). + * Calico Open Source was installed using the operator method. + Manifest- and Helm-based installations are not supported. +
+ Check Calico compatibility + + To confirm that you're on a compatible Calico setup, you can run the following command: + + ```bash + kubectl get tigerastatus goldmane + ``` + + If you're running Calico Open Source 3.30+ with the `Goldmane` custom resource is enabled, you should see something like this: + + ```bash title='Example output' + NAME AVAILABLE PROGRESSING DEGRADED SINCE + goldmane True False False 21h + ``` + This cluster is ready to connect to Calico Cloud Free Tier. + + If instead you get an error (`Error from server (NotFound): tigerastatuses.operator.tigera.io "goldmane" not found`), then you need to do one of the following: + * If you're already on 3.30+ [enable the `Goldmane` custom resource](/calico/latest/observability/enable-whisker). + * If you're on 3.29 or earlier, [upgrade your Calico installation to 3.30 or later](/calico/latest/operations/upgrading/kubernetes-upgrade#upgrading-an-installation-that-uses-the-operator) and then [enable the `Goldmane` custom resource](/calico/latest/observability/enable-whisker). +
+* You have an active Calico Cloud Free Tier account. + To create an account, go to the [Calico Cloud Free Tier sign-up page](https://calicocloud.io/?code=free). +* You are signed in to the $[prodname] web console. + +## Connect your cluster + +1. From the web console, click the **Connect a cluster** button on the welcome screen. +1. Follow the prompts to create a name for your cluster and copy a `kubectl` command to run in your cluster. + +
+ What's happening in this command? + + This command creates three resources in your cluster: + * **A `ManagementClusterConnection` resource**. + This resource specifies the address of the Calico Cloud management cluster. + * **A `Secret` resource (`tigera-managed-cluster-connection`)**. + This resource provides certificates for secure communication between your cluster and the Calico Cloud management cluster. + * **A `Secret` resource (`tigera-voltron-linseed-certs-public`)**. + This resource provides certificates for secure communications for the specific components that Calico Cloud uses for log data and observability. + + ```bash title='Example of generated kubectl command to connect a cluster to Calico Cloud Free Tier' + kubectl apply -f - < + +1. To start the connection process, run the `kubectl` command in your terminal. + + ```bash title='Sample output' + managementclusterconnection.operator.tigera.io/tigera-secure created + secret/tigera-managed-cluster-connection created + secret/tigera-voltron-linseed-certs-public created + ``` + +1. After your run the command, click **I applied the manifest** to continue. + +## Verify your connection + +* From the web console, click **Managed clusters** to view a list of managed clusters. + If your cluster connected successfully, you'll see its connection status is **Connected**. + +## Troubleshooting + +This section describes solutions to common connection problems. + +### Connection status is **Connected**, but there is no data in my dashboards or Service Graph + +#### Cause: The `Goldmane` resource is not enabled + +Goldmane is the component that serves the flow logs API. +Without it, your cluster may be able to connect to Calico Cloud Free Tier, but it won't be able to provide flow logs. + +To fix this problem: + +* [Enable `Goldmane` custom resource](/calico/latest/observability/enable-whisker). + + A few minutes after you apply the `Goldmane` resource, you will begin to see data in your dashboards and Service Graph. + +### Connection status remains **Disconnected** after applying the connection manifest + +#### Cause 1: Your cluster is running Calico Open Source 3.29 or earlier + +Clusters with Calico Open Source 3.29 or earlier can't connect to Calico Cloud Free Tier. + +After applying the connection manifest, you may get output like this: + +```bash +secret/tigera-managed-cluster-connection created +secret/tigera-voltron-linseed-certs-public created +error: resource mapping not found for name: "tigera-secure" namespace: "" from "STDIN": no matches for kind "ManagementClusterConnection" in version "operator.tigera.io/v1" +ensure CRDs are installed first +``` + +To fix this problem: + +1. [Upgrade your cluster to Calico Open Source 3.30 or later](/calico/latest/operations/upgrading/kubernetes-upgrade#upgrading-an-installation-that-uses-the-operator) and [enable the `Goldmane` resource](/calico/latest/observability/enable-whisker). +1. Reapply the connection manifest. + +#### Cause 2: Your network environment restricts egress traffic to the Calico Cloud management cluster + +If your network environment restricts egress traffic to the Calico Cloud management cluster, you need to create allow rules so your cluster can connect. + +To fix this problem: + +1. Locate the address of the Calico Cloud management cluster in the `ManagementClusterAddr` field of the `ManagementClusterConnection` resource: + + ```bash + kubectl get managementclusterconnection tigera-secure -o jsonpath='{.spec.managementClusterAddr}' + ``` + + You'll see something like this: + + ```bash title='Sample output' + oss-ui-01-management.calicocloud.io:443 + ``` + +1. Make sure your network allows TCP egress traffic to this address. + + +## Next steps + +* To try out the observability tools with a demo application, follow the quickstart guide from [Step 4: Deploy the demo app](quickstart.mdx#step-4-deploy-the-demo-app). +* [Remove a cluster from Calico Cloud Free Tier](disconnect-cluster-free.mdx) \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/free/disconnect-cluster-free.mdx b/calico-cloud_versioned_docs/version-24-1/free/disconnect-cluster-free.mdx new file mode 100644 index 0000000000..8871279d08 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/free/disconnect-cluster-free.mdx @@ -0,0 +1,42 @@ +--- +description: Disconnect a Kubernetes cluster from Calico Cloud Free Tier and remove the Calico Cloud components it installed. +title: Remove a cluster +--- + +# Remove a cluster from Calico Cloud Free Tier + +You can disconnect and remove your cluster from Calico Cloud Free Tier in minutes. +When you remove your cluster, you will no longer be able to view or manage it from the Calico Cloud web console. +All log data will be permanently deleted within 24 hours. + +## Prerequisites + +* You have a cluster that is connected to Calico Cloud Free Tier. +* You are signed in to the $[prodname] web console. +* You have `kubectl` access to the connected Kubernetes cluster. + +## Remove a cluster from Calico Cloud Free Tier + +1. Disconnect your cluster from Calico Cloud Free Tier by deleting the three resources you created to connect your cluster: + + ```bash + kubectl delete managementclusterconnection tigera-secure + kubectl delete secret -n tigera-operator tigera-managed-cluster-connection + kubectl delete secret -n tigera-operator tigera-voltron-linseed-certs-public + ``` + ```bash title='Sample output' + managementclusterconnection.operator.tigera.io "tigera-secure" deleted + secret "tigera-managed-cluster-connection" deleted + secret "tigera-voltron-linseed-certs-public" deleted + ``` + + On the **Managed Clusters** page, your cluster's connection status will change to **Disconnected**. + + + +1. To remove your cluster from Calico Cloud, click **Actions > Delete**. +1. If you made changes to your network policies or firewall to allow egress traffic to Calico Cloud, you may want to revert those changes. + +## Next steps + +* [Connect a cluster to Calico Cloud Free Tier](connect-cluster-free.mdx) \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/free/index.mdx b/calico-cloud_versioned_docs/version-24-1/free/index.mdx new file mode 100644 index 0000000000..9fd29836b2 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/free/index.mdx @@ -0,0 +1,14 @@ +--- +description: Connect a Kubernetes cluster to Calico Cloud Free Tier — a no-cost path to centralized network observability without a paid Calico Cloud subscription. +--- + +import { DocCardLink, DocCardLinkLayout } from '/src/___new___/components'; + +# Calico Cloud Free Tier + + + + + + + \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/free/overview.mdx b/calico-cloud_versioned_docs/version-24-1/free/overview.mdx new file mode 100644 index 0000000000..e99e0984d1 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/free/overview.mdx @@ -0,0 +1,99 @@ +--- +description: What Calico Cloud Free Tier includes and excludes versus paid Calico Cloud — limits, supported platforms, and the upgrade path to a paid subscription. +title: Overview +--- + +import Screenshot from '/src/___new___/components/Screenshot'; +export let figCount = 1; + +# Calico Cloud Free Tier + +Learn about the observability tools available to you when you connect your Calico Open Source cluster to the Calico Cloud Free Tier web console. + +## About Calico Cloud Free Tier + +Calico Cloud Free Tier provides essential observability tools for understanding and securing your Kubernetes cluster. +With these tools, you can monitor network traffic, visualize service interactions, and gain insights into the behavior of your network policies. + +A free account with Calico Cloud allows you to connect one cluster to the free-tier web console. +The cluster must be running Calico Open Source 3.30 or later. + +Features available in the free-tier web console are similar to those in the pro-tier console, but with some limitations. +For example, you can view your network policies, but you cannot edit them directly in the console, as you can with Calico Cloud Pro. +As a general rule, the free-tier console allows you to view what's going on in your cluster. +To make changes, such as creating or editing policies, you must use `kubectl` commands in your cluster. + +### What happens when you connect your cluster? + +Calico Open Source 3.30 introduced a new component, Goldmane, which exposes a gRPC API with aggregated flow logs from your cluster. +This is the data source that feeds [Calico Whisker](/calico/latest/observability/view-flow-logs), an in-cluster web console that displays a stream of flows. + +Calico Cloud Free Tier uses the same flow logs API to receive data from your cluster. +When you connect your cluster to Calico Cloud Free Tier, you apply manifests that establish a secure connection to the Calico Cloud management cluster. +These manifests create a `ManagementClusterConnection` resource in your cluster, which specifies the address of the Calico Cloud management cluster, and two `Secret` resources that provide certificates for secure communication. + +Unlike Whisker, which can only display a stream of flows in real time, Calico Cloud Free Tier collects and retains flow logs for a 24-hour period. +You can sign in to the console at any time to view data from the last 24 hours. +If you choose to disconnect and remove your cluster from the console, your data is deleted after a day. + +And because you're not installing anything complicated in your cluster, you can easily remove the resources and restore you cluster to its previous state. + +## Feature overview + +Calico Cloud Free Tier provides a set of observability features that help you monitor and manage your cluster's network traffic. + +### Dashboards + +Dashboards help you see and understand the activity in your cluster using categorized graphs and charts. +They visually represent data from your logs, providing a birds-eye view of cluster health, traffic volume, DNS activity, and performance metrics. +This allows for quick assessment of the overall state of your cluster's network. + + +*Figure {figCount++}: The traffic volume dashboard* + +### Service Graph + +Service Graph provides a point-to-point, topographical representation of network traffic within your cluster. +It serves as the primary tool for visualizing how services communicate and for troubleshooting connectivity issues. +You can click on nodes or connections to see detailed information and associated logs for targeted analysis. + + +*Figure {figCount++}: The Service Graph page.* + +### Policies + +Network policy is the primary tool for securing a Kubernetes network by controlling traffic flow. +Policies restrict network traffic (egress and ingress) in your cluster so only the traffic that you explicitly allow can flow. +Calico Cloud supports standard Kubernetes policies and its own enhanced network policies for fine-grained control. + + +*Figure {figCount++}: The Policies page.* + +For more information, see the Calico Open Source [network policy documentation](/calico/latest/network-policy/). + +### Policy recommendations + +Policy recommendations help you secure your Kubernetes environments by automatically suggesting network policies based on observed network traffic. +This proactively identifies necessary communication flows and enables you to implement a least-privilege security model with minimal effort. +The recommendations are generated by analyzing flow logs. +You can activate the recommendations as staged network policies for testing before you use them in production. + + +*Figure {figCount++}: The Policy Recommendations page.* + +### Network sets + +Network sets are resources for defining IP subnetworks or CIDRs, which can then be easily referenced in network policies using standard label selectors. +They are a powerful feature for simplifying policy management, especially when dealing with external network ranges or grouping common IP addresses. +Using network sets makes policies more reusable and scalable. + +For more information, see: +* [Understanding network sets](../tutorials/calico-cloud-features/networksets.mdx). +* Calico Open Source resource documentation on [Network set](/calico/latest/reference/resources/networkset) and [Global network set](/calico/latest/reference/resources/globalnetworkset). + +## Next steps + +* [Sign up for a free account with Calico Cloud](https://calicocloud.io/?code=free). +* [Tutorial: Quickstart guide to Calico Cloud Free Tier](quickstart.mdx). +* [Connect a cluster to Calico Cloud Free Tier](connect-cluster-free.mdx) +* For an overview of the expanded feature set in Calico Cloud Pro, take a [tour of the full web console](../tutorials/calico-cloud-features/tour.mdx). \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/free/quickstart.mdx b/calico-cloud_versioned_docs/version-24-1/free/quickstart.mdx new file mode 100644 index 0000000000..6f42333fda --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/free/quickstart.mdx @@ -0,0 +1,174 @@ +--- +description: Quickstart that connects a Kubernetes cluster to Calico Cloud Free Tier for centralized network observability — no payment or trial required. +title: Quickstart +--- + +import Screenshot from '/src/___new___/components/Screenshot'; +export let figCount = 1; + +# Calico Cloud Free Tier quickstart guide + +Get Calico Cloud running on a local Kind cluster, deploy a realistic microservices app, and explore your live Service Graph — all in a few minutes. + +## Before you begin + +* [Sign up for a Calico Cloud Free Tier account](https://calicocloud.io/?code=free). +* Install [Kind](https://kind.sigs.k8s.io/docs/user/quick-start/), [Docker](https://docs.docker.com/desktop/), and [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl). + +## Step 1: Create a Kind cluster + +1. Create a file called `kind-config.yaml`: + + ```yaml + kind: Cluster + apiVersion: kind.x-k8s.io/v1alpha4 + nodes: + - role: control-plane + - role: worker + - role: worker + networking: + disableDefaultCNI: true + podSubnet: 192.168.0.0/16 + ``` + + This tells Kind to create a three-node cluster without a default CNI, so you can install Calico in the next step. + +1. Create the cluster: + + ```bash + kind create cluster --name=calico-cluster --config=kind-config.yaml + ``` + +1. Verify the nodes are up (they'll show `NotReady` until Calico is installed): + + ```bash + kubectl get nodes + ``` + + ```bash title="Expected output" + NAME STATUS ROLES AGE VERSION + calico-cluster-control-plane NotReady control-plane 60s v1.29.2 + calico-cluster-worker NotReady 40s v1.29.2 + calico-cluster-worker2 NotReady 40s v1.29.2 + ``` + +## Step 2: Install Calico + +1. Install the Tigera operator and custom resource definitions: + + ```bash + kubectl create -f $[manifestsUrl]/manifests/tigera-operator.yaml + ``` + +1. Install $[prodname] by creating the necessary custom resources: + + ```bash + kubectl create -f $[manifestsUrl]/manifests/custom-resources.yaml + ``` + +1. Wait for all components to become available: + + ```bash + watch kubectl get tigerastatus + ``` + + After a few minutes, all components should show `True` in the `AVAILABLE` column. + + ```bash title="Expected output" + NAME AVAILABLE PROGRESSING DEGRADED SINCE + apiserver True False False 4m9s + calico True False False 3m29s + goldmane True False False 3m39s + ippools True False False 6m4s + whisker True False False 3m19s + ``` + +## Step 3: Connect to Calico Cloud Free Tier + +1. In the Calico Cloud web console, click **Connect a cluster**. +1. Follow the prompts to name your cluster (for example, `quickstart-cluster`) and copy the generated `kubectl` command. + +
+ What's happening in this command? + + This command creates three resources in your cluster: + * **A `ManagementClusterConnection` resource**. + This resource specifies the address of the Calico Cloud management cluster. + * **A `Secret` resource (`tigera-managed-cluster-connection`)**. + This resource provides certificates for secure communication between your cluster and the Calico Cloud management cluster. + * **A `Secret` resource (`tigera-voltron-linseed-certs-public`)**. + This resource provides certificates for secure communications for the specific components that Calico Cloud uses for log data and observability. +
+ +1. Run the `kubectl` command in your terminal. +1. Back in the console, click **I applied the manifest**. + When the **Managed Clusters** page shows your cluster as **Connected**, move to the next step. + +## Step 4: Deploy the demo app + +This step deploys [Google's Online Boutique](https://github.com/GoogleCloudPlatform/microservices-demo), a microservices demo with 12 services. +Each service is deployed in its own namespace so that Service Graph displays a rich cross-namespace topology with realistic traffic patterns. + +1. Deploy the application: + + ```bash + kubectl apply -f $[tutorialFilesURL]/online-boutique-namespaced.yaml + ``` + + :::note + + The manifest includes a load generator that automatically drives traffic between all services. + No manual traffic generation is needed. + + ::: + +1. Wait for all pods to reach `Running` status: + + ```bash + watch "kubectl get pods -A | grep -v Running | grep -v Completed" + ``` + + This may take 2–3 minutes. When the only output is the header line, all pods are ready. + +
+ What's in the demo app? + + The Online Boutique is a cloud-native e-commerce app made up of 12 microservices: + `adservice`, `cartservice`, `checkoutservice`, `currencyservice`, `emailservice`, `frontend`, `loadgenerator`, `paymentservice`, `productcatalogservice`, `recommendationservice`, `redis-cart`, and `shippingservice`. + + Each service runs in its own namespace. + A built-in load generator continuously sends requests through the frontend, creating realistic traffic across all services. +
+ +## Step 5: Explore Service Graph + +In the Calico Cloud web console, go to **Service Graph**. +Within a minute or two you'll see every service and namespace mapped as a live node, with directional traffic flows between them. + +Try clicking on a service — for example, `cartservice`. You'll see: +* Which services send traffic to it (such as `checkoutservice` and `frontend`) +* Which services it connects to (such as `redis-cart`) +* Flow logs with details on allowed and denied connections + + +*Figure {figCount++}: Service Graph showing cross-namespace traffic flows between the Online Boutique microservices.* + +## Step 6: Clean up + +1. Delete the Kind cluster: + + ```bash + kind delete cluster --name=calico-cluster + ``` + +1. In the Calico Cloud web console, go to **Managed Clusters**, click **Actions > Disconnect**, and then click **I ran the commands**. + (Because you already deleted the cluster, you don't need to run the disconnect commands.) + +1. Click **Actions > Remove** to fully remove the cluster from Calico Cloud. + +## Next steps + +* [Write a network policy](/calico-cloud/network-policy/beginners/calico-network-policy) to restrict traffic between services — for example, allow only `cartservice` to reach `redis-cart`. +* [Set up alerts](/calico-cloud/observability/alerts) for unexpected traffic patterns. +* [Explore the full feature set](/calico-cloud/free/overview) available on the Free Tier. +* [Connect another cluster](/calico-cloud/free/connect-cluster-free) to Calico Cloud Free Tier. diff --git a/calico-cloud_versioned_docs/version-24-1/get-help/support.mdx b/calico-cloud_versioned_docs/version-24-1/get-help/support.mdx new file mode 100644 index 0000000000..deafae328a --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/get-help/support.mdx @@ -0,0 +1,33 @@ +--- +description: Support and feedback channels for Calico Cloud users, including the Tigera support portal, in-console help, status page, and the Calico Cloud support policy. +--- + +import IconUser from '/img/icons/user-icon.svg'; + +# Support and feedback + +## Contact Support + +You can find solutions to many common problems by following our [troubleshooting checklist](../get-started/checklist.mdx) or by consulting the [Tigera Help Center](https://www.tigera.io/calico-support/). + +For everything else, you can open a support ticket. + +### Paid Calico Cloud users + +Sign in to the [support portal](https://tigeraio.my.site.com/community/s/login/) to open a ticket. + +### Free trial users + +From the web console, click the user icon > **Contact Support**, and then complete the form. + +## Provide feedback + +We value your feedback and suggestions for improvement. Email us: help@calicocloud.io. + +## Support policy + +For details, see our [Support policy](https://www.tigera.io/legal/calico-cloud-support-policy). + +## Check the status of Calico Cloud services + +Go to [Calico Cloud Status](https://status.calicocloud.io) to view the current status of our sites and services. \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/get-started/cc-arch-diagram.mdx b/calico-cloud_versioned_docs/version-24-1/get-started/cc-arch-diagram.mdx new file mode 100644 index 0000000000..d15723edd0 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/get-started/cc-arch-diagram.mdx @@ -0,0 +1,45 @@ +--- +description: Architecture overview of Calico Cloud — components that run in the connected cluster and the SaaS-side services they communicate with. +--- + +# Calico Cloud architecture + +## $[prodname] security + +$[prodname] architecture is based on the $[prodname] multi-cluster management feature. $[prodname] manages the control plane, and you connect your clusters (called **managed clusters**) to the control plane. Communication between the $[prodname] control plane and managed clusters is secured using TLS tunnels. + +![calico-architecture](/img/calico-cloud/cc-architecture.svg) + +The components that secure communications between the $[prodname] control plane and managed clusters are: + +- **$[prodname] tunnel server** - accepts secure TLS connections from managed clusters +- **Guardian** - an agent that runs in each managed cluster that proxies communication between $[prodname] components and managed cluster components + +All connections go through the $[prodname] tunnel server and Guardian. The only exception is during installation and upgrade when managed clusters connect to $[prodname] using TLS connections to get install/update resources, register the cluster, and report the status of the install/update. + +The $[prodname] tunnel is initiated by Guardian on the managed cluster. The $[prodname] control plane does not initiate new connections to the managed cluster outside of the tunnel. However, there are connections that go through the $[prodname] tunnel server that are initiated from the control plane; for example, when a user interacts with the web console, or when configuration needs to be pushed into the managed cluster. + +## Managed cluster components + +The following diagram shows the major components in a managed cluster, followed by component descriptions. + +![calico-architecture-diagram](/img/calico-cloud/cc-arch-diagram.png) + +| Component | Description | Ports/Protocol | +| ------------------------------ | ------------------------------------------------------------ | ------------------------------------------------------------ | +| $[prodname] controller | Deploys required resources for $[prodname]. | TCP 6443 to Kubernetes API server | +| $[prodname] installer | Gets installation resources from the $[prodname] portal, registers a managed cluster, and reports installation or upgrade progress. | • TCP 443 to $[prodname] hosted service
• TCP 6443 to Kubernetes API server | +| $[prodname] tunnel server | Communicates with managed clusters by creating secure TLS tunnels. | Port 9000 from managed clusters | +| calico-node | Bundles key components that are required for networking containers with $[prodname]:

• Felix
• BIRD
• confd | • TCP 5473 to Typha
• TCP 9900 and 9081 from Prometheus API service | +| Fluentd | Open-source data collector for unified logging. Collects and forwards $[prodname] logs (flows, DNS, L7) to log storage. | • TCP 8080 to Guardian
• TCP 9080 from Prometheus API service | +| Guardian | An agent running in each managed cluster that proxies communication between the $[prodname] tunnel server and your managed cluster. Secured using TLS tunnels.
| • Port 9000 to tunnel server
• TCP 6443 to Kubernetes API server
• TCP 6443 from $[prodname] components | +| Installation endpoints | Endpoints at `*.calicocloud.io` and `*.projectcalico.org`. | TCP 443 for both | +| Intrusion detection controller | Handles integrations with threat intelligence feeds and $[prodname] custom alerts. | • TCP 8080 to Guardian
• TCP 6443 to Kubernetes API server | +| Kubernetes API server | A Kubernetes component that validates and configures data for the API objects (for example, pods, services, and others).
| TCP 6443 (from all components) | +| kube-controllers | Monitors the Kubernetes API and performs actions based on cluster state. $[prodname] kube-controllers container includes these controllers:

• Node
• Service
• Federated services
• Authorization
| • TCP 9094 from Prometheus API service
• TCP 6443 to Kubernetes API server | +| Log storage | Storage for logs (flows, L7, DNS, audit). Data for each managed cluster is isolated and protected against unauthorized access. | n/a | +| Packet capture API | Retrieves capture files (pcap format) generated by a packet capture for use with network protocol analysis tools like Wireshark. Packet capture data is visible in the web console and Service Graph. | • TCP 8449 Guardian to Packet Capture API
• TCP 6443 to Kubernetes API server | +| Prometheus API service | Collects metrics from $[prodname] components and makes the metrics available to the web console. | • TCP 6443 to Kubernetes API server
• TCP 9080 to Fluentd
• TCP 9900 and 9081 to Prometheus API service | +| Tigera API server | Allows users to manage $[prodname] resources such as policies and tiers through kubectl or the Kubernetes API server. | • TCP 9095 to Prometheus API service
• TCP 8080 from Kubernetes API server | +| Typha | Increases scale by reducing each node’s impact on the datastore. | TCP 5473 from calico-node to Typha | +| User access to the web console | Authenticated users can access the browser-based the web console, which provides network traffic visibility and troubleshooting, centralized multi-cluster management, threat-defense, and policy lifecycle management for multiple roles/stakeholders. | Port 443 to $[prodname] tunnel server | \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/get-started/checklist.mdx b/calico-cloud_versioned_docs/version-24-1/get-started/checklist.mdx new file mode 100644 index 0000000000..7186a494f5 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/get-started/checklist.mdx @@ -0,0 +1,82 @@ +--- +description: Gather information and run pre-flight checks before opening a Calico Cloud support ticket so triage moves quickly. +--- + +# Troubleshooting checklist + +## Check $[prodname] installation + +Installing $[prodname] on your Kubernetes cluster is managed by the $[prodname] operator. The $[prodname] operator is deployed as a Deployment in the `calico-cloud` namespace, and records status in a custom resource named `installer`. + +Check the `installer` status using the following command. + +```bash +kubectl get installer default --namespace calico-cloud -o jsonpath --template '{.status}' +``` + +**Sample output** + +``` +{"clusterName":"my-new-cluster","state":"installing"} +``` + +After `state` is `complete`, $[prodname] is properly installed. + +## Check logs for fatal errors + +Check that the $[prodname] operator is running and that logs do not have any fatal errors. + +```bash +kubectl logs -n calico-cloud deployment/calico-cloud-controller-manager +2022-04-04T14:34:32.472Z INFO controller-runtime.metrics metrics server is starting to listen {"addr": "127.0.0.1:8080"} +2022-04-04T14:34:32.472Z INFO setup starting manager +2022-04-04T14:34:32.472Z INFO setup config {"ccBaseURL": "https://www.dev.calicocloud.io/api", "debug": false, "leader-elect": true} +I0404 14:34:32.472586 1 leaderelection.go:243] attempting to acquire leader lease calico-cloud/c2ad41ce.calicocloud.io... +2022-04-04T14:34:32.472Z INFO controller-runtime.manager starting metrics server {"path": "/metrics"} +I0404 14:34:32.480870 1 leaderelection.go:253] successfully acquired lease calico-cloud/c2ad41ce.calicocloud.io +<...> +``` + +## Check custom resources + +Verify that you have the **installer custom resource**, and that the values are appropriate for your environment. + +```bash +kubectl get installers.operator.calicocloud.io --namespace calico-cloud -o yaml +``` + +```yaml +apiVersion: v1 +items: + - apiVersion: operator.calicocloud.io/v1 + kind: Installer + metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"operator.calicocloud.io/v1","kind":"Installer","metadata":{"annotations":{},"name":"my-new-cluster","namespace":"calico-cloud"}} + creationTimestamp: '2022-04-04T14:34:29Z' + generation: 1 + name: my-new-cluster + namespace: calico-cloud + resourceVersion: '1102' + uid: eb1d1cd0-f01f-47b2-81fe-8eee46dbe712 + status: + clusterName: my-new-cluster + message: '' + resourceVersion: '' + state: installing +kind: List +metadata: + resourceVersion: '' + selfLink: '' +``` + +## Send failed installation diagnostics to Calico Cloud support + +To upload diagnostics about a failed installation to $[prodname] support, run the following command: + +```bash +kubectl patch installer -n calico-cloud default --type merge -p='{"spec":{"uploadDiags":true}}' +``` + + diff --git a/calico-cloud_versioned_docs/version-24-1/get-started/connect-cluster.mdx b/calico-cloud_versioned_docs/version-24-1/get-started/connect-cluster.mdx new file mode 100644 index 0000000000..e4e4045eb6 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/get-started/connect-cluster.mdx @@ -0,0 +1,66 @@ +--- +description: What happens when you connect a Kubernetes cluster to Calico Cloud — what is installed, what data leaves the cluster, and what changes in the cluster. +--- + +# What happens when you connect a cluster to Calico Cloud + +Although connecting your cluster to $[prodname] is easy, we also understand you may want details about what happens when your cluster is managed by $[prodname]. We hope this article, along with the other topics in this section, gives you the information you need to install $[prodname] with confidence. If you have other questions, let us know: [Support policy](https://www.tigera.io/legal/calico-cloud-support-policy) or email: feedback@calicocloud.io. + +## What happens when you connect your cluster + +- Your cluster is registered and connected to $[prodname] +- Your Calico open source install is updated with resources for $[prodname] features +- Tigera components and services are added to collect metrics and logs that are sent to the Calico Cloud management plane, which provides the basis for visibility and troubleshooting +- Policies are added to secure communications between Tigera components +- A global threat feed is added to alert on any egress traffic to addresses in the threat feed to protect the cluster +- All of your existing network policies (Kubernetes and Calico) can be found in a single place: the default tier + +After your cluster is connected, it is listed in the Managed Clusters page where you can move between multiple clusters. + +![managed-clusters](/img/calico-cloud/managed-clusters.png) + +## What happens when you disconnect your cluster + +Whether you’ve finished your $[prodname] Trial, or terminated your licensed $[prodname] subscription, we know you want your cluster to remain functional. $[prodname] provides a migration script that returns your cluster to a working state in open-source Calico. The migration script: + +- Removes and cleans up all $[prodname] components and services that have no equivalent in open-source Calico +- Switches the operator configuration to open-source Calico, which migrates your cluster to a version of open-source Calico +- Ensures all Calico policies are migrated to the default tier, or allows you to remove all Calico policies + +For details of the migration script, see [Uninstall $[prodname] from a cluster](../operations/disconnect.mdx). + +## What happens under the covers when you connect your cluster + +### Pre-check + +- Verifies that your cluster can be migrated, and validates that the cluster platform and version is supported by $[prodname] +- Records the number of nodes and other basic attributes of your cluster. This data remains on your system and is used for troubleshooting in case there are issues connecting your cluster. + +### Install and connect + +- Based on the Calico install for your cluster, the manifest is upgraded/migrated to use the $[prodname] Tigera Operator. (Note that clusters installed using Helm are currently not supported.) +- Tigera Operator installs the required $[prodname] Custom Defined Resources (CRDs), and a standard pull secret to allow access to $[prodname] images +- Installs a Prometheus instance for component metrics (if it doesn’t already exist on the cluster) +- Pushes a license to the cluster so components receive appropriate entitlements +- Adds custom namespaces, service account, and role bindings to support cluster access and permissions to the $[prodname] user interface +- Creates RBAC permissions to access these resources: + - Number of nodes in the cluster and stats for billing purposes + - Policies in the cluster, and pod information +- Registers the cluster so it can connect to $[prodname] +- Creates a global threat feed to alert on any egress traffic to addresses in the threat feed +- Adds policies to secure communication for $[prodname] components, including safeguards that prevent any new network policies from impacting the vital functions of the cluster +- Creates roles and bindings to allow the installer to operate and to allow Calico Cloud to operate after installation. + Each of these objects is given minimal permissions for its specific function. +- Creates roles and bindings for two user types for access to the web console: + - **Admin** - full permissions for network policies and $[prodname] resources, and read permissions for namespace and pods + - **User** - read-only/view permissions to same resources above + +## How long does it take to connect a cluster? + +Typically, about 5 minutes. + +## Troubleshooting an installation + +- A checklist to [troubleshoot your installation](checklist.mdx) + +- Additional troubleshooting for the [Tigera Operator](operator-checklist.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/get-started/index.mdx b/calico-cloud_versioned_docs/version-24-1/get-started/index.mdx new file mode 100644 index 0000000000..f35b054054 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/get-started/index.mdx @@ -0,0 +1,43 @@ +--- +description: Connect Kubernetes clusters to Calico Cloud and upgrade them — the entry point for the Calico Cloud onboarding flow. +hide_table_of_contents: true +--- + +import { DocCardLink, DocCardLinkLayout } from '/src/___new___/components'; + +# Install and upgrade + +Requirements and guides for connecting your Kubernetes cluster to Calico Cloud. + +## Before you begin + + + + + + + + + +## Connect your cluster + + + + + + + + + +## Troubleshooting + + + + + + +## Upgrade + + + + \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/get-started/install-automated.mdx b/calico-cloud_versioned_docs/version-24-1/get-started/install-automated.mdx new file mode 100644 index 0000000000..8d302efd65 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/get-started/install-automated.mdx @@ -0,0 +1,190 @@ +--- +description: Connect a Kubernetes cluster to Calico Cloud as part of an automated CI or provisioning workflow rather than the interactive UI flow. +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import IconUser from '/img/icons/user-icon.svg'; + +# Install Calico Cloud as part of an automated workflow + +You can connect clusters to Calico Cloud as part of an automated workflow, using persistent client credentials and customized Helm charts. + +## Prerequisites + +* You have an paid Calico Cloud account. +* You are signed in to the web console as a user with the **Owner** or **Admin**, role. +* You have at least one cluster that meets our [system requirements](system-requirements.mdx). +* You have kubectl access to the cluster. +* You have installed Helm 3.0 or later on your workstation. + +## Create client credentials + +Create client credentials and generate a Kubernetes secret to use for automated Helm installations. + +1. Select the user icon **> Settings**. +1. Under the **Client Credentials** tab, click **Add Client Credential** +1. In the **Add Client Credential** dialog, enter a name and click **Create**. + Your new client credential will appear in the list on the **Manage Client Credentials** page. +1. Locate the newly created client credential in the list and select **Action** > **Manage keys** > **Add Key** +1. Enter a name, choose how long the key will be valid, and click **Create key**. +1. Click **Download** to download the `.yaml` secret file and store it in a secure location. + You will not be able to retrieve this secret again. + +:::important + +To ensure that you always have a valid key, you should transition to a second key before the first key expires. +Create a second key, download the secret, and then replace copies of the secret file for the first key with the secret file for the second key. +When all the secrets from the first key have been replaced, you can safely delete the first key from the **Client Credentials** page. +When the key is deleted, all API requests based on that key will be rejected. + +::: + +## About customizing your Helm installation + +You can customize your Calico Cloud installation for the following purposes: + +* to enable or disable certain features +* to modify pod scheduling and resource management + +To do this, you can either edit the default `values.yaml` file or pass individual key-value pairs using the `--set` flag for the `helm upgrade` command. + +### Required parameters + +The following parameters are required for all Calico Cloud installations. + +| Parameter | Value | Example | Description | +| -- | -- | -- | -- | +| `installer.clusterName` | string | `cluster-name` | The name given to your managed cluster in Calico Cloud. | +| `installer.calicoCloudVersion` | string | `$[cloudUserVersion]` | The version of Calico Cloud you're installing. | + + +```yaml title="Example from values.yaml with clusterName and calicoCloudVersion" +installer: + clusterName: example-cluster + calicoCloudVersion: $[cloudUserVersion] +``` + +### Optional parameters for private registries + +If you're using a private registry, you must set the following parameters. + +| Parameter | Value | Example | Description | +| -- | -- | -- | -- | +| `installer.registry` | string | `my.registry/` | The name of your private registry. | +| `installer.imagePath` | string | `path/to/directory` | The path to a directory in your private registry that contains images required to install Calico Cloud. | +| `imagePullSecrets.name` | string | `secret-name` | The name of the image pull secret you will use to allow access from the `calico-cloud` namespace to your private registry. | + +### Optional parameters for features + +The following parameters enable certain features in Calico Cloud. +These features can be enabled or disabled only by setting them in your `values.yaml` file at installation. + +| Feature name | Parameter | Values | +|---------|-----|--------| +| Packet Capture | `installer.components.packetCaptureAPI.state` | `Enabled`, `Disabled` (default) | + +:::note + +If you're upgrading from Calico Cloud 19, the Packet Capture features will remain enabled unless you explicitly set them to `Disabled`. + +::: + +### Optional parameters for pod scheduling and resource management + +For many Calico Cloud components, you can specify node selectors, tolerations, and resource requests and limits. +The full list of Calico Cloud components is available in the default `values.yaml` file. + +:::note +Helm may overwrite previous customizations of custom resource fields available under the `installer.components` Helm parameter. +For `installer.components`, you should define all your `values.yaml` customizations to be sure nothing is lost during Calico Cloud upgrades and reinstalls. +::: + +## Prepare your values.yaml with customizations + +***Prerequisites*** +* You reviewed the information about available customizations in [About customizing your Helm installation](#about-customizing-your-helm-installation). +* If you're installing from a private registry, you [added the Calico Cloud images to a private registry](setup-private-registry.mdx), and you have the following information about the registry: + * Registry secret name + :::note + If your private registry requires credentials, create a `calico-cloud` namespace on your cluster. + Then, create an image pull secret and use this name for the **Registry Secret Name**. + ::: + * Image registry + * Image path + +1. Add the Calico Cloud Helm repository to your local client: + + ```bash + helm repo add calico-cloud https://installer.calicocloud.io/charts --force-update + ``` + +1. Save the default values definitions to your workstation so you can edit them locally: + ```bash + helm show values calico-cloud/calico-cloud > .yaml + ``` + All editable values are provided in the default values definitions. + +1. Add values for the required parameters, `installer.clusterName` and `installer.calicoCloudVersion`. + + ```yaml title="Example from values.yaml file with clusterName and calicoCloudVersion" + installer: + clusterName: example-cluster + calicoCloudVersion: $[cloudUserVersion] + ``` + +1. Add values for the optional parameters. + For each resource you want to edit, uncomment the object, add a value, and save. + + ```yaml title="Example from values.yaml file with Packet Capture enabled" + installer: + components: + packetCaptureAPI: + state: Enabled + ``` + +## Install Calico Cloud as part of an automated workflow + +You can install Calico Cloud using repeatable kubectl or Helm commands together with valid client credentials. +These commands can be added to any automated workflow. + +***Prerequisites*** + +* You have generated a set of client credentials and you know the path to your secret. +* You have a `values.yaml` file with your customizations. + +1. Add the Calico Cloud Helm repository to your local client. + + ```bash + helm repo add calico-cloud https://installer.calicocloud.io/charts --force-update + ``` + +1. Add the Calico Cloud custom resource definitions: + + ```bash + helm upgrade --install calico-cloud-crds calico-cloud/calico-cloud-crds \ + --namespace calico-cloud \ + --create-namespace + ``` + +1. Apply the client credentials secret to your cluster. + + ```bash + kubectl apply -f + ``` + + :::important + You should keep track of this with a secret management system. + ::: + +1. Apply the Calico Cloud installer custom resource with your customizations in the `values.yaml` file. + + ```bash + helm upgrade --install calico-cloud calico-cloud/calico-cloud \ + --namespace calico-cloud \ + -f .yaml + ``` + +## Additional resources + +* [Calico Cloud installation reference](https://docs.tigera.io/calico-cloud/reference/installation/api) \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/get-started/install-cluster.mdx b/calico-cloud_versioned_docs/version-24-1/get-started/install-cluster.mdx new file mode 100644 index 0000000000..59365dd3e1 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/get-started/install-cluster.mdx @@ -0,0 +1,64 @@ +--- +description: Connect a Kubernetes cluster to Calico Cloud using the standard install command from the management UI. +title: Install Calico Cloud +--- + +# Connect a cluster to Calico Cloud + +You can quickly connect a cluster to Calico Cloud by generating a unique kubectl or Helm command in the web console and running it on your cluster. + +## Prerequisites + +* You have an paid Calico Cloud account. +* You are signed in to the web console as a user with the **Owner**, **Admin**, or **DevOps** role. +* You have at least one cluster that meets our [system requirements](system-requirements.mdx). +* You have kubectl access to the cluster. +* If you're using Helm, you installed Helm 3.0 or later on your workstation. + +## Connect a cluster to Calico Cloud with kubectl + +1. If your organization uses multiple projects to group managed clusters, click the **Project** menu and select the project you want your cluster to be part of. +1. From the **Managed Clusters** page, click **Connect Cluster**. +1. In the **Connect a Cluster** dialog, enter a **Cluster Name** and select a **Cluster Type**. +1. Optional: If you must install a specific older release, select the Calico Cloud version you want to install. + We always recommend the latest version, which is installed by default. +1. Click **Connect** to generate a unique kubectl command. Copy the command. +1. From a terminal, paste and run the command. +1. On the **Managed Clusters** page, you should immediately see your cluster in the list of managed clusters. + Monitor the status under **Connection Status**. + When the status changes to **Connected**, installation is complete and your cluster is connected to Calico Cloud. + +## Connect a cluster to Calico Cloud with Helm + +1. If your organization uses multiple projects to group managed clusters, click the **Project** menu and select the project you want your cluster to be part of. +1. From the **Managed Clusters** page, click **Connect Cluster**. +1. In the **Connect a Cluster** dialog, enter a **Cluster Name** and select a **Cluster Type**. +1. Optional: If you must install a specific older release, select the Calico Cloud version you want to install. + We always recommend the latest version, which is installed by default. +1. Click **Connect** to generate a unique Helm installation command. Copy the command. + ```bash title="Example of generated Helm installation command" + helm repo add calico-cloud https://installer.calicocloud.io/charts --force-update && helm upgrade --install calico-cloud-crds calico-cloud/calico-cloud-crds --namespace calico-cloud --create-namespace && helm upgrade --install calico-cloud calico-cloud/calico-cloud --namespace calico-cloud --set apiKey=ryl34elz8:9dav6eoag:ifk1uwruwlgp7vzn7ecijt5zjbf5p9p1il1ag8877ylwjo4muu19wzg2g8x5qa7x --set installer.clusterName=my-cluster --set installer.calicoCloudVersion=v19.1.0 + ``` +1. Optional: If you want to enable the Packet Capture feature, you can append `--set installer.components.packetCaptureAPI.state=Enabled` to the generated Helm command. + You can change this option only by reinstalling or upgrading Calico Cloud and changing the values. + | Feature | Key | Values | + |---------|-----|--------| + | Packet Capture | `installer.components.packetCaptureAPI.state` | `Enabled`, `Disabled` (default\*)
* The default for new clusters is `Disabled`. For upgrades for previously connected clusters, the default will retain the previous state. | + + + ```bash title="Example of generated Helm command with user-added parameters" + helm repo add calico-cloud https://installer.calicocloud.io/charts --force-update && helm upgrade --install calico-cloud-crds calico-cloud/calico-cloud-crds --namespace calico-cloud --create-namespace && helm upgrade --install calico-cloud calico-cloud/calico-cloud --namespace calico-cloud --set apiKey=ryl34elz8:9dav6eoag:ifk1uwruwlgp7vzn7ecijt5zjbf5p9p1il1ag8877ylwjo4muu19wzg2g8x5qa7x --set installer.clusterName=my-cluster --set installer.calicoCloudVersion=v19.1.0 \ + --set installer.components.packetCaptureAPI.state=Enabled + ``` + + In this example, the command connects the cluster to Calico Cloud with the Packet Capture feature enabled. + +1. From a terminal, paste and run the command. +1. On the **Managed Clusters** page, you should immediately see your cluster in the list of managed clusters. + Monitor the status under **Connection Status**. + When the status changes to **Connected**, installation is complete and your cluster is connected to Calico Cloud. + +## Additional resources + +* [Calico Cloud troubleshooting checklist](checklist.mdx) +* [Tigera Operator troubleshooting checklist](operator-checklist.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/get-started/install-private-registry.mdx b/calico-cloud_versioned_docs/version-24-1/get-started/install-private-registry.mdx new file mode 100644 index 0000000000..90df29c75f --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/get-started/install-private-registry.mdx @@ -0,0 +1,59 @@ +--- +description: Connect a Kubernetes cluster to Calico Cloud when its container images must be pulled from a private registry. +title: Install using a private registry +--- + +# Connect a cluster to Calico Cloud using a private registry + +You can perform a Helm installation from images stored on a private registry. + +## Prerequisites + +* You have an paid Calico Cloud account. +* You are signed in to the web console as a user with the **Owner**, **Admin**, or **DevOps** role. +* You have at least one cluster that meets our [system requirements](system-requirements.mdx). +* You have kubectl access to the cluster. +* You have installed Helm 3.0 or later on your workstation. +* You have [added the Calico Cloud images to a private registry](setup-private-registry.mdx), and you have the following information about the registry: + * Registry secret name + :::note + If your private registry requires credentials, create a `calico-cloud` namespace on your cluster. + Then, create an image pull secret and use this name for the **Registry Secret Name**. + ::: + * Image registry + * Image path + + +## Install Calico Cloud using a private registry + +1. If your organization uses multiple projects to group managed clusters, click the **Project** menu and select the project you want your cluster to be part of. +1. From the **Managed Clusters** page, click **Connect Cluster**. +1. In the **Connect a Cluster** dialog, enter a **Cluster Name** and select a **Cluster Type**. +1. Optional: If you must install a specific older release, select the Calico Cloud version you want to install. We always recommend the latest version, which is installed by default. +1. Click **Advanced Options**, and then select both **Install via Helm** and **Private registry**. +1. Enter the **Registry Secret Name**, **Image registry**, and **Image path**. +1. Click **Connect** to generate a unique Helm installation command. Copy the command. +1. Optional: If you want to enable the Packet Capture feature, you can append `--set installer.components.packetCaptureAPI.state=Enabled` to the generated Helm command. + You can change this option only by reinstalling or upgrading Calico Cloud and changing the values. + + | Feature | Key | Values | + |---------|-----|--------| + | Packet Capture | `installer.components.packetCaptureAPI.state` | `Enabled`, `Disabled` (default\*)
* The default for new clusters is `Disabled`. For upgrades for previously connected clusters, the default will retain the previous state. | + + ```bash title="Example of generated Helm command with user-added parameters" + helm repo add calico-cloud https://installer.calicocloud.io/charts --force-update && helm upgrade --install calico-cloud-crds calico-cloud/calico-cloud-crds --namespace calico-cloud --create-namespace && helm upgrade --install calico-cloud calico-cloud/calico-cloud --namespace calico-cloud --set apiKey=ryl34elz8:9dav6eoag:ifk1uwruwlgp7vzn7ecijt5zjbf5p9p1il1ag8877ylwjo4muu19wzg2g8x5qa7x --set installer.clusterName=my-cluster --set installer.calicoCloudVersion=v19.1.0 \ + --set installer.components.packetCaptureAPI.state=Enabled + ``` + + In this example, the command connects the cluster to Calico Cloud with the Packet Capture feature enabled. + +1. From a terminal, paste and run the command. + +1. On the **Managed Clusters** page, you should immediately see your cluster in the list of managed clusters. + Monitor the status under **Connection Status**. + When the status changes to **Connected**, installation is complete and your cluster is connected to Calico Cloud. + +## Additional resources + +* [Calico Cloud troubleshooting checklist](checklist.mdx) +* [Tigera Operator troubleshooting checklist](operator-checklist.mdx) \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/get-started/operator-checklist.mdx b/calico-cloud_versioned_docs/version-24-1/get-started/operator-checklist.mdx new file mode 100644 index 0000000000..6063d98a32 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/get-started/operator-checklist.mdx @@ -0,0 +1,644 @@ +--- +description: Troubleshoot the Tigera Operator on Calico Cloud connected clusters when the standard support checklist is not enough. +--- + +# Tigera Operator troubleshooting checklist + +If you have issues getting your cluster up and running, use this checklist. + +- [Check installation start errors](#check-installation-start-errors) +- [Check Calico Cloud installation](#check-calico-cloud-installation) +- [Check logs for fatal errors](#check-logs-for-fatal-errors) +- [Check custom resources](#check-custom-resources) +- [Check pod capacity](#check-pod-capacity) +- [Check the web console dashboard for traffic](#check-the-web-console-dashboard-for-traffic) + +## Check installation start errors + +Are you seeing any of these issues at the start of installation? + +### [ERROR] Detected plugin ls: No such file or directory, it is currently not supported + +The cluster you are using to install $[prodname] does not have a CNI plugin installed, the CNI is incompatible. If your cluster has functional pod networking and you see this message, it is likely that kubelet has been configured to use kubenet networking, which is not compatible with $[prodname]. You can use a different cluster, or re-create your cluster with [compatible networking](system-requirements.mdx). + +### $[prodname] cannot be connected to a cluster with FIPS mode enabled + +At this time, FIPS mode is not supported in $[prodname]. Disable FIPS mode in the cluster and install again. + +### Install script is taking a long time + +If you are migrating a large cluster from a previous manifest-based Calico install, the script can take some time; this is normal. + +But, it could also mean that your cluster has an incompatibility. Go to the next step [Check Calico Cloud installation](#check-calico-cloud-installation). + +## Check Calico Cloud installation + +Installing $[prodname] on your Kubernetes cluster is managed by the Tigera Operator. The Tigera Operator is deployed as a ReplicaSet in the `tigera-operator` namespace, and records status in a custom resource named, `tigerastatus`. The operator get its configuration from several Custom Resources (CRs); the central one is the Installation CR. + +Check `tigerastatus` using the following command. + +```bash +kubectl get tigerastatus +``` + +**Sample output** + +``` +NAME AVAILABLE PROGRESSING DEGRADED SINCE +apiserver True False False 10m +calico True False False 11m +cloud-core True False False 11m +compliance True False False 9m39s +intrusion-detection True False False 9m49s +log-collector True False False 9m29s +management-cluster-connection True False False 9m54s +monitor True False False 10m +``` + +If all components show a status of "Available" = TRUE, $[prodname] is properly installed. + +**Issue: $[prodname] is not installed** + +If $[prodname] is not installed, you'll get the following error. Install $[prodname] on the node using the `curl` command that you got from Support. + +```bash +kubectl get tigerastatus +error: the server doesn't have a resource type "tigerastatus" +``` + +**Issue: $[prodname] components are missing or are degraded** + +If some of the $[prodname] components are Available = FALSE or DEGRADED = TRUE, run the following command and contact Support with the following output. + +```bash +kubectl get tigerastatus -o yaml +``` + +:::note + +If you are using the **AWS or Azure CNI plugin**, a degraded state is likely because you do not have enough pod capacity on your nodes. To fix this, see [Check pod capacity](#check-pod-capacity). + +::: + +**Sample output** + +In the following example, the typha component has an issue because it is showing `AVAILABLE: FALSE`, and `DEGRADED: TRUE`. To understand details of $[prodname] components, see [Deep dive into custom resources](#deep-dive-into-custom-resources). + +```yaml +apiVersion: v1 +items: + - apiVersion: operator.tigera.io/v1 + kind: TigeraStatus + metadata: + creationTimestamp: '2020-12-30T17:13:30Z' + generation: 1 + managedFields: + - apiVersion: operator.tigera.io/v1 + fieldsType: FieldsV1 + fieldsV1: + f:spec: {} + f:status: + .: {} + f:conditions: {} + manager: operator + operation: Update + time: '2020-12-30T17:16:20Z' + name: calico + resourceVersion: '8166' + selfLink: /apis/operator.tigera.io/v1/tigerastatuses/calico + uid: 39a8a2d0-2074-418c-b52d-0baa0a48f4a1 + spec: {} + status: + conditions: + - lastTransitionTime: '2020-12-30T17:13:30Z' + status: 'False' + type: Available + - lastTransitionTime: '2020-12-30T17:13:30Z' + message: DaemonSet "calico-system/calico-node" is not yet scheduled on any nodes + reason: Not all pods are ready + status: 'True' + type: Progressing + - lastTransitionTime: '2020-12-30T17:13:30Z' + message: 'failed to wait for operator typha deployment to be ready: waiting + for typha to have 4 replicas, currently at 3' + reason: error migrating resources to calico-system + status: 'True' + type: Degraded +kind: List +metadata: + resourceVersion: '' + selfLink: '' +``` + +## Check logs for fatal errors + +Check that the Tigera Operator is running and that logs do not have any fatal errors. + +```bash +kubectl get pods -n tigera-operator +``` + +``` +NAME READY STATUS RESTARTS AGE +tigera-operator-8687585b66-68gmr 1/1 Running 0 139m +``` + +```bash +kubectl logs -n tigera-operator tigera-operator-8687585b66-68gmr +``` + +``` +2020/12/30 17:38:54 [INFO] Version: 90975f4 +2020/12/30 17:38:54 [INFO] Go Version: go1.14.4 +2020/12/30 17:38:54 [INFO] Go OS/Arch: linux/amd64 +{"level":"info","ts":1609349935.2848425,"logger":"setup","msg":"Checking type of cluster","provider":""} +{"level":"info","ts":1609349935.2868738,"logger":"setup","msg":"Checking if TSEE controllers are required","required":true} +<...> +``` + +## Check custom resources + +Verify that you have the **installation custom resource**, and that the values are appropriate for your environment. + +```bash +kubectl get installation.operator.tigera.io default -o yaml +``` + +```yaml +apiVersion: operator.tigera.io/v1 +kind: Installation +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"operator.tigera.io/v1","kind":"Installation","metadata":{"annotations":{},"name":"default"},"spec":{"imagePullSecrets":[{"name":"tigera-pull-secret"}],"variant":"CalicoEnterprise"}} + creationTimestamp: '2021-01-20T19:50:23Z' + generation: 2 + managedFields: + - apiVersion: operator.tigera.io/v1 + fieldsType: FieldsV1 + fieldsV1: + f:metadata: + f:annotations: + .: {} + f:kubectl.kubernetes.io/last-applied-configuration: {} + f:spec: + .: {} + f:imagePullSecrets: {} + f:variant: {} + manager: kubectl + operation: Update + time: '2021-01-20T19:50:23Z' + - apiVersion: operator.tigera.io/v1 + fieldsType: FieldsV1 + fieldsV1: + f:spec: + f:calicoNetwork: + .: {} + f:bgp: {} + f:hostPorts: {} + f:ipPools: {} + f:mtu: {} + f:multiInterfaceMode: {} + f:nodeAddressAutodetectionV4: + .: {} + f:firstFound: {} + f:cni: + .: {} + f:ipam: + .: {} + f:type: {} + f:type: {} + f:componentResources: {} + f:flexVolumePath: {} + f:nodeUpdateStrategy: + .: {} + f:rollingUpdate: + .: {} + f:maxUnavailable: {} + f:type: {} + f:status: + .: {} + f:variant: {} + manager: operator + operation: Update + time: '2021-01-20T19:55:10Z' + name: default + resourceVersion: '5195' + selfLink: /apis/operator.tigera.io/v1/installations/default + uid: 016c3f0b-39f0-48a0-9da8-a59a81ed9128 +spec: + calicoNetwork: + bgp: Enabled + hostPorts: Enabled + ipPools: + - blockSize: 26 + cidr: 10.42.0.0/16 + encapsulation: IPIP + natOutgoing: Enabled + nodeSelector: all() + mtu: 0 + multiInterfaceMode: None + nodeAddressAutodetectionV4: + firstFound: true + cni: + ipam: + type: Calico + type: Calico + componentResources: + - componentName: Node + resourceRequirements: + requests: + cpu: 250m + flexVolumePath: /usr/libexec/kubernetes/kubelet-plugins/volume/exec/nodeagent~uds + imagePullSecrets: + - name: tigera-pull-secret + nodeUpdateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + variant: CalicoEnterprise +status: + variant: CalicoEnterprise +``` + +Verify that you have the following custom resources. In the default installation, there is no configuration information. + +**Check API server** + +```bash +kubectl get apiserver.operator.tigera.io tigera-secure +``` + +``` +NAME AGE +tigera-secure 85m +``` + +**Check cloud core** + +```bash +kubectl get cloudcore.operator.tigera.io tigera-secure +``` + +``` +NAME AGE +tigera-secure 88m +``` + +**Check compliance** + +```bash +kubectl get compliance.operator.tigera.io tigera-secure +``` + +``` +NAME AGE +tigera-secure 90m +``` + +**Check intrusion detection** + +```bash +kubectl get intrusiondetection.operator.tigera.io tigera-secure +``` + +``` +NAME AGE +tigera-secure 93m +``` + +**Check log collector** + +```bash +kubectl get logcollector.operator.tigera.io tigera-secure +``` + +``` +NAME AGE +tigera-secure 96m +``` + +**Check management cluster** + +```bash +kubectl get ManagementClusterConnection.operator.tigera.io tigera-secure -o yaml +``` + +```yaml +apiVersion: operator.tigera.io/v1 +kind: ManagementClusterConnection +metadata: + annotations: + kubectl.kubernetes.io/last-applied-configuration: | + {"apiVersion":"operator.tigera.io/v1","kind":"ManagementClusterConnection","metadata":{"annotations":{},"name":"tigera-secure"},"spec":{"managementClusterAddr":".tigera.io:9000"}} + creationTimestamp: '2021-01-20T19:55:40Z' + generation: 1 + managedFields: + - apiVersion: operator.tigera.io/v1 + fieldsType: FieldsV1 + fieldsV1: + f:metadata: + f:annotations: + .: {} + f:kubectl.kubernetes.io/last-applied-configuration: {} + f:spec: + .: {} + f:managementClusterAddr: {} + manager: kubectl + operation: Update + time: '2021-01-20T19:55:40Z' + name: tigera-secure + resourceVersion: '5425' + selfLink: /apis/operator.tigera.io/v1/managementclusterconnections/tigera-secure + uid: b7a2093e-a4b6-4e76-b291-15f45bfa11cf +spec: + managementClusterAddr: .tigera.io:9000 +``` + +**Check monitor** + +```bash +kubectl get monitor.operator.tigera.io tigera-secure +``` + +``` +NAME AGE +tigera-secure 98m +``` + +For more information on operator custom resources see the [Tigera Operator API reference](../reference/installation/api.mdx). + +### Deep dive into custom resources + +Run the following command to see if you have required custom resources: + +```bash +kubectl get tigerastatus +``` + +| | NAME | AVAILABLE | PROGRESSING | DEGRADED | SINCE | +| --- | ----------------------------- | --------- | ----------- | -------- | ----- | +| 1 | apiserver | TRUE | FALSE | FALSE | 10m | +| 2 | calico | TRUE | FALSE | FALSE | 11m | +| 3 | cloud-core | TRUE | FALSE | FALSE | 11m | +| 4 | compliance | TRUE | FALSE | FALSE | 9m39s | +| 5 | intrusion-detection | TRUE | FALSE | FALSE | 9m49s | +| 6 | log-collector | TRUE | FALSE | FALSE | 9m29s | +| 7 | management-cluster-connection | TRUE | FALSE | FALSE | 9m54s | +| 8 | monitor | TRUE | FALSE | FALSE | 11m | + +**1 - api server** + +`apiserver` is a required component that is an aggregated api-server. It is required for things like applying the tigera license. If `tigerastatus` reports it as unavailable or degraded, check the pods and logs in the `calico-system`namespace. For example, + +```bash +kubectl get pods -n calico-system +``` + +``` +NAME READY STATUS RESTARTS AGE +calico-apiserver-5c75bc8d4b-sbn6g 2/2 Running 0 45m +``` + +**2 - calico** + +`calico` is the core component for networking. If it is not available or degraded, check the pods and their logs in the `calico-system` namespace. There should be a calico-node pod running on each of your nodes. You should have at least one `calico-typha` pod and the number will scale with the number of nodes in your cluster. You should have a `calico-kube-controllers` pod running. For example, + +```bash +kubectl get pods -n calico-system +``` + +``` +NAME READY STATUS RESTARTS AGE +calico-kube-controllers-5c77d4d559-hfl5d 1/1 Running 0 44m +calico-node-6s2c9 1/1 Running 0 40m +calico-node-8nf28 1/1 Running 0 41m +calico-node-djlrg 1/1 Running 0 40m +calico-node-ms8nv 1/1 Running 0 40m +calico-node-t7pck 1/1 Running 0 40m +calico-typha-bdb494458-76gcx 1/1 Running 0 41m +calico-typha-bdb494458-847tr 1/1 Running 0 41m +calico-typha-bdb494458-k8lhj 1/1 Running 0 40m +calico-typha-bdb494458-vjbjz 1/1 Running 0 40m +``` + +**3 - cloud-core** + +`cloud-core` is responsible for predefined and custom roles for users. Check the pods and logs in the `calico-cloud` namespace with the label selector `k8s-app=cc-core-operator`. + +```bash +$ kubectl get pods -n calico-cloud -l k8s-app=cc-core-operator +``` + +``` +NAME READY STATUS RESTARTS AGE +cc-core-operator-126dcd494a-9kj7g 1/1 Running 0 80m +``` + +**4 - compliance** + +`compliance` manages the compliance components that still run on your cluster, such as the CIS benchmarker. +Check the pods and logs in the `tigera-compliance` namespace. + +```bash +$ kubectl get pods -n tigera-compliance +``` + +``` +NAME READY STATUS RESTARTS AGE +compliance-benchmarker-bqvps 1/1 Running 0 65m +compliance-benchmarker-h58hr 1/1 Running 0 65m +compliance-benchmarker-kdtwp 1/1 Running 0 65m +compliance-benchmarker-mzm2z 1/1 Running 0 65m +compliance-benchmarker-s5mmf 1/1 Running 0 65m +compliance-controller-77785646df-ws2cj 1/1 Running 0 65m +compliance-snapshotter-6bcbdc65b-66k9v 1/1 Running 0 65m +``` + +**5 - intrusion-detection** + +`intrusion-detection` is responsible for the intrusion detection features. Check the pods and logs in the `tigera-intrusion-detection` namespace. + +```bash +$ kubectl get pods -n tigera-intrusion-detection +``` + +``` +NAME READY STATUS RESTARTS AGE +intrusion-detection-controller-669bf45c75-grvz9 1/1 Running 0 66m +intrusion-detection-es-job-installer-xm22v 1/1 Running 0 66m +``` + +**6 - log-collector** + +`log-collector` collects flow and other logs and forwards them to $[prodname]. Check the pods and logs in the `tigera-fluentd` namespace. You should have one pod running on each of your nodes. + +```bash +kubectl get pods -n tigera-fluentd +``` + +``` +NAME READY STATUS RESTARTS AGE +fluentd-node-5mzh6 1/1 Running 0 70m +fluentd-node-7vmxw 1/1 Running 0 70m +fluentd-node-bbc4p 1/1 Running 0 70m +fluentd-node-chfz4 1/1 Running 0 70m +fluentd-node-d6f56 1/1 Running 0 70m +``` + +**7 - management-cluster-connection** + +The `management-cluster-connection` is required for your managed clusters to connect to the $[prodname] backend. If it is not available or degraded, check the pods and logs in the `tigera-guardian` namespace. + +```bash +kubectl get pods -n tigera-guardian +``` + +``` +NAME READY STATUS RESTARTS AGE +tigera-guardian-7d5d94d5cc-49rg8 1/1 Running 0 48m +``` + +To verify that the guardian component has network connectivity to the management cluster: + +Find the URL to the management cluster: + +```bash +kubectl get managementclusterconnection tigera-secure -o=jsonpath='{.spec.managementClusterAddr}' +.tigera.io:9000 +``` + +then, from a worker node, verify network connectivity to the management cluster: + +```bash +openssl s_client -connect .tigera.io:9000 +``` + +``` +CONNECTED(00000003) +depth=0 CN = tigera-voltron +verify error:num=18:self signed certificate +verify return:1 +depth=0 CN = tigera-voltron +verify return:1 +--- +Certificate chain + 0 s:CN = tigera-voltron + i:CN = tigera-voltron +--- +Server certificate +-----BEGIN CERTIFICATE----- +MIIC5DCCAcygAwIBAgIBATANBgkqhkiG9w0BAQsFADAZMRcwFQYDVQQDEw50aWdl +cmEtdm9sdHJvbjAeFw0yMDEyMjExOTA1MzhaFw0yNTEyMjAxOTA1MzhaMBkxFzAV +<...> +``` + +**8 - monitor** + +`monitor` is responsible for configuring prometheus and associated custom resources. Check the pods and logs in the `tigera-prometheus` namespace. + +```bash +$ kubectl get pods -n tigera-prometheus +``` + +``` +NAME READY STATUS RESTARTS AGE +alertmanager-calico-node-alertmanager-0 2/2 Running 0 125m +alertmanager-calico-node-alertmanager-1 2/2 Running 0 125m +alertmanager-calico-node-alertmanager-2 2/2 Running 0 125m +calico-prometheus-operator-77bf897c9b-7f88x 1/1 Running 0 125m +prometheus-calico-node-prometheus-0 3/3 Running 1 125m +``` + +### Check additional custom resources + +Check for the presence of other custom resources created by the Tigera Operator: FelixConfiguration, IPPool, Tigera License, and Prometheus for component metrics. + +`FelixConfiguration` contains configuration that are not configured as environment variables to the`calico-node` container. + +```bash +kubectl get Felixconfiguration default +``` + +``` +NAME CREATED AT +default 2021-01-20T19:49:35Z +``` + +The operator creates a default `IPPool` for your pod networking if it does not already exist; in this case, the CIDR is taken from the Installation CR. + +```bash +kubectl get IPPool +``` + +``` +NAME CREATED AT +default-ipv4-ippool 2021-01-20T19:49:35Z +``` + +A Tigera license is applied by the installation script. + +```bash +kubectl get licensekeys.projectcalico.org +``` + +``` +NAME AGE +default 120m +``` + +The installation script deploys a Prometheus operator and associated custom resources. If you already have a Prometheus operator running in your cluster, contact Tigera support. + +```bash +kubectl get pods -n tigera-prometheus +``` + +``` +NAME READY STATUS RESTARTS AGE +alertmanager-calico-node-alertmanager-0 2/2 Running 0 125m +alertmanager-calico-node-alertmanager-1 2/2 Running 0 125m +alertmanager-calico-node-alertmanager-2 2/2 Running 0 125m +calico-prometheus-operator-77bf897c9b-7f88x 1/1 Running 0 125m +prometheus-calico-node-prometheus-0 3/3 Running 1 125m +``` + +## Check pod capacity + +If cluster does not have enough capacity, it will not be able to deploy pods. There is no specific error associated with this condition. + +The high-level components $[prodname] needs to run are: + +- Per node: 1 fluentd, 1 compliance benchmarker +- On top of per node: 3 alertmanager (from statefulset), 1 prometheus, 1 prometheus operator, 1 kube-controllers, 2 compliance snapshotter and controller, 1 guardian, 1 ids controller, 1 apiserver + +Some clusters have limited pod-networked pod capacity. + +- For the AWS CNI plugin, the [number of pods that can be networked is based on the size of the instance](https://docs.aws.amazon.com/eks/latest/userguide/pod-networking.html). +- For AKS with the Azure CNI, the [number of pods that can be networked is set at cluster deployment time or when new node pools are created (default of 30)](https://docs.microsoft.com/en-us/azure/aks/configure-azure-cni#maximum-pods-per-node). +- For GKE, there is a [hard limit of 110 pods per node](https://docs.cloud.google.com/kubernetes-engine/docs/concepts/planning-scalability). +- For Calico CNI, the pod limit is based on the available IPs in the IPPool, and there is no specific per node limit. + +Verify you have the following pod-networked pod capacity. + +- Verify on each node in your cluster that there is capacity for at least 2 pods. +- Verify there is capacity for at least 11 pods in the cluster in addition to the per node capacity. + +To check the capacity of individual nodes on AWS or AKS, query the node status and look at `Capacity.Pods` (which is the total capacity for the node). To get the number of pod-networked pods for a node, count the pods on the node that are pod-networked (non-hostNetworked pods). + +## Check pod security policy violation + +If your cluster is using Kubernetes version 1.24 or earlier, a pod security policy (PSP) violation may be blocking pods on the cluster. + +Search for the term `PodSecurityPolicy` in the status message of failed cluster deployments. If a PSP is present, install open source Calico in the cluster before you connect to $[prodname]. + +## Check the web console dashboard for traffic + +### Web console main dashboard is missing traffic graphs + +When you log in to the web console, the first item in the left nav is the main Dashboard. This dashboard is a birds-eye view of your managed cluster activity for policy and networking. For the graphs to display traffic, you must have the Prometheus operator. If it is missing, the following message is displayed during installation: + +`Prometheus Operator detected in the cluster. Skipping Tigera Prometheus Operator` + +To install an appropriate Prometheus operator, contact Support below. + + diff --git a/calico-cloud_versioned_docs/version-24-1/get-started/prepare-cluster.mdx b/calico-cloud_versioned_docs/version-24-1/get-started/prepare-cluster.mdx new file mode 100644 index 0000000000..ba852b3dd6 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/get-started/prepare-cluster.mdx @@ -0,0 +1,78 @@ +--- +description: Prepare a Kubernetes cluster for connection to Calico Cloud — pre-flight checks, RBAC, and image-pull configuration. +--- + +# Prepare your cluster for Calico Cloud + +Get your cluster ready to connect to Calico Cloud. + +## Prerequisites + +* Your cluster meets the [system requirements](system-requirements.mdx) for Calico Cloud. + +## Allow outbound traffic from pods to Calico Cloud endpoints + +Pods running in your Kubernetes cluster must allow outbound traffic to the following endpoints: + +- `https://installer.calicocloud.io:443/*` +- `https://www.calicocloud.io:443/api/*` +- `https://client-auth.calicocloud.io:443/*` +- TCP to `.calicocloud.io:9000` + +For each node, Docker must be able to pull images from the following endpoints: + +* `quay.io` +* `cdn01.quay.io` +* `cdn02.quay.io` +* `us-docker.pkg.dev` + +## Make sure you have the right permissions for your platform user account + +If your cluster is installed on a managed service, you must have sufficient permissions from your identity and access management system. +Check that you are authorized to create the following Kubernetes resource types: + +* `ClusterRole` +* `ClusterRoleBinding` +* `Deployment` +* `ServiceAccount` +* `CustomResourceDefinition` + +## Prepare your cluster on Azure Kubernetes Service + +### Remove taints from Linux node pools + +If you have a hybrid clusters with both Windows and Linux nodes, the Linux nodes may have taints that prevent Calico Cloud from scheduling pods on those nodes. +These taints must be removed before you connect your cluster to Calico Cloud. + +You can check whether any node pools in your cluster have taints by running the following command: + +```bash +az aks nodepool list --resource-group --cluster-name --query "[].{name:name nodeTaints:nodeTaints}" +``` + +Remove any taints in the Linux node pools by running the command: + +```bash +az aks nodepool update --resource-group --cluster-name --name --node-taints "" +``` + +## Prepare your cluster Google Kubernetes Engine + +### Turn on intranode visibility for your cluster + +Verify that intranode visibility is set to `Enabled` by running the following command: + +```bash +gcloud container clusters describe --flatten networkConfig.enableIntraNodeVisibility +``` + +If intranode visibility is not enabled, you must enable it by following running the following command: + +```bash +gcloud container clusters update --enable-intra-node-visibility +``` + +## Next steps + +* [Connect your cluster to Calico Cloud](install-cluster.mdx) +* [Connect your cluster using a private registry](install-private-registry.mdx) \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/get-started/setup-private-registry.mdx b/calico-cloud_versioned_docs/version-24-1/get-started/setup-private-registry.mdx new file mode 100644 index 0000000000..054f945842 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/get-started/setup-private-registry.mdx @@ -0,0 +1,103 @@ +--- +description: Mirror Calico Cloud container images into a private registry so air-gapped clusters can install without reaching the public registry. +--- + +# Set up a private registry + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import VersionedCloudImageList from '@site/src/___new___/components/VersionedCloudImageList'; + +## Big picture + +Add $[prodname] images to a private registry to install new clusters or update existing ones. + +## Value + +In some deployments, installing $[prodname] in clusters from third-party repos is not an option. Before you can install $[prodname] from a private registry, you must first add $[prodname] images to the registry. + +## Concepts + +A **container image registry** (often known as a **registry**), is a service where you can push, pull, and store container images. In Kubernetes, a registry is considered _private_ if it is not publicly available. + +A **private registry** is not publicly available. It must be accessible on a private network or with an **image pull secret** for authenticated access. + +An **image path** is a directory in a registry that contains images required to install $[prodname]. + +## Before you begin + +**Required** + +- [Helm CLI](https://helm.sh/docs/intro/install/) command +- Use the [Crane command](https://github.com/google/go-containerregistry/blob/main/cmd/crane/README.md) to follow steps in this guide. (Other tools can be used, but commands be adjusted accordingly.) +- Image registry + - Must be private (accessible only on a private network or using log in credentials) + - Images copied to the registry must not be made publicly available + - Push access to your registry + +## How to + +Add the required $[prodname] images to a private registry. + +### Set up registry credentials + +1. Log into $[prodname] and navigate to "Managed Clusters". +1. If your organization uses multiple projects to group managed clusters, click the **Project** menu and select the project you want your cluster to be part of. +1. Get the "Registry Credentials" by clicking on the icon. ![registry credentials](/img/calico-cloud/private-registry-icon.png) +1. Apply the credentials so the $[prodname] images can be accessed. + +### Create the list of required images + + + +### Set up the private registry + +```bash +REGISTRY= +``` + +If you want all images to come from the same path in your registry, set this image path value. Otherwise unset this environment variable or set it to the empty string. + +```bash +IMAGEPATH="" +``` + +#### Image examples + +| Original image | Image Registry | Image Path | Private registry image | +| ----------------------------------- | -------------- | ------------- | -------------------------------------------- | +| `quay.io/tigera/typha:v1.2.3` | `my.registry/` | | `my.registry/tigera/typha:v1.2.3` | +| `quay.io/tigera/typha:v1.2.3` | `my.registry/` | `custom-path` | `my.registry/custom-path/typha:v1.2.3` | +| `quay.io/tigera/cc-operator:v4.5.6` | `my.registry/` | | `my.registry/tigera/cc-operator:v4.5.6` | +| `quay.io/tigera/cc-operator:v4.5.6` | `my.registry/` | `custom-path` | `my.registry/custom-path/cc-operator:v4.5.6` | + +### Copy images to your registry + +For $[prodname] to install images from your registry, copy the images from the standard registries into your own registry. + + + + +```bash +for image in ${IMAGES[@]}; do + img_base=$(echo ${image} | sed "s#^.*/\([^/]*/[^/]*$\)#\1#") + crane cp ${image} ${REGISTRY}${img_base} || break +done +``` + + + + +```bash +for image in ${IMAGES[@]}; do + img_base=$(echo ${image} | sed "s#^.*/##") + crane cp ${image} ${REGISTRY}${IMAGEPATH}/${img_base} || break +done +``` + + + + +## Install using the private registry + +Follow the directions [to connect a cluster to Calico Cloud](install-cluster.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/get-started/system-requirements.mdx b/calico-cloud_versioned_docs/version-24-1/get-started/system-requirements.mdx new file mode 100644 index 0000000000..75baaeddc5 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/get-started/system-requirements.mdx @@ -0,0 +1,194 @@ +--- +description: Cluster, platform, and version requirements a Kubernetes cluster must meet before it can connect to Calico Cloud. +--- + +# System requirements + +Before you connect your cluster to Calico Cloud, make sure your cluster meets the system requirements. +Your cluster must already have a CNI installed before you can connect to Calico Cloud. + +{/* This file is referenced in the web console and its location should not be changed with accounting for references from pendo/manager UI. A symlink so this file exists at install/system-requirements. */} + +## Kubernetes distributions and CNIs + +Calico Cloud works with Kubernetes on self-provisioned infrastructure and on managed Kubernetes distributions. +To use Calico Cloud for both networking and network policy, your cluster must have Calico Open Source installed before you connect to Calico Cloud. +For most managed distributions, you can use the provider's CNI for networking and use Calico Cloud for network policy. + +| Distribution | Supported CNIs | +| --- | --- | +| Kubernetes on self-provisioned infrastructure | - Calico Open Source 3.20 or later | +| Amazon Elastic Kubernetes Service | - Calico Open Source 3.20 or later
- Amazon VPC CNI | +| Azure Kubernetes Service | - Calico Open Source 3.20 or later
- Azure CNI | +| Google Kubernetes Engine | - Calico Open Source 3.20 or later
- GKE CNI | +| Rancher Kubernetes Engine 2 | - Calico Open Source 3.20 or later | + +:::note + +The Kubernetes distributions listed above are those that Tigera currently tests and supports for Calico Cloud. +You may be able to connect clusters on other distributions with Calico Open Source installed as the CNI. +For more information about connecting other cluster types to Calico Cloud, [contact Support](https://tigeraio.my.site.com/community/s/login/). + +::: + +### Other distributions + +The Calico Cloud installer may be able to successfully connect + +## Kubernetes versions + +To install Calico Cloud $[cloudUserVersion], your Kubernetes distribution must be based on one of the following Kubernetes versions: + +* Kubernetes 1.36 +* Kubernetes 1.35 +* Kubernetes 1.34 + +
+ Kubernetes compatibility for older versions of Calico Cloud + + | Calico Cloud version | Kubernetes versions | + |-----------------------|---------------------| + | 23.0.0 | 1.34-1.36 | + | 22.5.0 | 1.31-1.35 | + | 22.4.1 | 1.31-1.35 | + | 22.4.0 | 1.31-1.34 | + | 22.3.0 | 1.31-1.34 | + | 22.2.0 | 1.31-1.34 | + | 22.1.0 | 1.31-1.34 | + | 22.0.1 | 1.31-1.33 | + | 22.0.0 | 1.31-1.33 | + | 21.3.0 | 1.31-1.33 | + | 21.2.0 | 1.30-1.31 | + | 21.1.0 | 1.30-1.31 | + | 21.0.0 | 1.30-1.31 | + | 20.4.0 | 1.29-1.31 | + | 20.3.1 | 1.29-1.30 | + | 20.2.0 | 1.28-1.30 | + | 20.1.0 | 1.28-1.30 | + | 20.0.1 | 1.28-1.30 | + | 20.0.0 | 1.28-1.30 | + +
+ +:::tip + +You can connect a cluster that uses a more recent (and unsupported) version of Kubernetes, but only for testing and development purposes. +Connecting such a cluster is not suitable for production or any other workloads. +To connect a cluster with an unsupported version of Kubernetes, start by following the standard installation procedure. +When the installation fails, follow the instructions in the error message to connect your cluster. + +::: + +## Architectures + +Calico Cloud can be installed on nodes based on the following chip architectures: +* x86-64 +* ARM64 + +## Windows node support for hybrid clusters + +Calico Cloud supports hybrid Kubernetes clusters that use a Linux control plane with Windows nodes. +You can install Calico Cloud on clusters of this type on the following managed distributions: +* Amazon Elastic Kubernetes Service +* Azure Kubernetes Service + +## Browser support for the web console + +To access the web console, you can use latest two versions of the following web browsers: +* Chrome +* Safari + +## Kubernetes reconcilers + +* $[prodname] cannot be usually be installed on clusters that are managed by any kind of Kubernetes reconciler (for example, the Kubernetes `addon-manager` component). To verify, look for an annotation called `addonmanager.kubernetes.io/mode` on either of the following resources. (The resources may not exist). + + * `tigera-operator` deployment in the `tigera-operator` namespace + * `calico-node` daemonset in the `kube-system` namespace + + If the following command finds addonmanager on either of the resources, then the `addon-manager` component is being used. Find a different cluster to use. + + ```bash + kubectl get -n -o yaml | grep ' addonmanager.kubernetes.io/mode:' + ``` +* Some AKS clusters with the `addon-manager` component are compatible with Calico Cloud. + If output from the following command includes "EnsureExists", then the install is compatible with $[prodname]. + + ```bash + kubectl get CustomResourceDefinition installations.operator.tigera.io -o yaml | grep ' addonmanager.kubernetes.io/mode:' + ``` + + :::note + + If the command output does not include "EnsureExists" and you are on a recent version of AKS your cluster might still be compatible. + You can [contact Support](https://tigeraio.my.site.com/community/s/login/) for more information. + + ::: + + :::warning + + If your cluster already has Calico installed by AKS and managed by the `addon-manager` component, the standard [uninstall](../operations/disconnect.mdx) + is not supported. You will need to reach out to support to create a plan to uninstall $[prodname]. + + ::: + + +## Distribution-specific requirements + +### Azure Kubernetes Service + +* Your cluster uses a supported combination of the `networkPlugin` and `networkPolicy` configurations: + - `"networkPlugin": "none"` and `"networkPolicy": null` + - `"networkPlugin": "azure"` and `"networkPolicy": null` + - `"networkPlugin": "azure"` and `"networkPolicy": "calico"` + + You can check your configuration by running the following command: + + ```bash + az aks show --query 'networkProfile' + ``` + +* If your cluster uses the Azure CNI, your cluster's CNI is set to [transparent mode](https://docs.microsoft.com/en-us/azure/aks/faq#what-is-azure-cni-transparent-mode-vs-bridge-mode). + + ```bash + az vmss run-command invoke -g -n --scripts "cat /etc/cni/net.d/*" --command-id RunShellScript --instance-id 0 --query 'value[0].message' + ``` + + If the Azure CNI is enabled, the output should include `"mode": "transparent"`. + +### Google Kubernetes Service + +* Your cluster's network policy is disabled. + To verify, run the following command: + + ```bash + gcloud container clusters describe --flatten addonsConfig.networkPolicyConfig.disabled + ``` + +{/* vale CalicoStyle.Substitutions = NO */} +* Your cluster's Dataplane V2 is set to `null`. +{/* vale CalicoStyle.Substitutions = YES */} + To verify, run the following command: + + ```bash + gcloud container clusters describe --flatten networkConfig.datapathProvider + ``` + +### Rancher Kubernetes Engine 2 + +* The Calico Open Source CNI must not be provisioned by the RKE2 installer. + You can connect an RKE2 cluster to Calico Cloud only if: + * the RKE2 cluster was installed without a CNI + * Calico Open Source was installed manually + + To verify, run this command to see the configuration on your control-plane node: + + ```bash + cat /etc/rancher/rke2/config.yaml + ``` + You should see `cni: none`. + + If you're creating a new RKE2 cluster, you can set this configuration as an environment variable (`RKE2_CNI=none`) when you run the installation script. + +## Next steps + +* [Prepare your cluster for Calico Cloud](prepare-cluster.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/get-started/upgrade-cluster.mdx b/calico-cloud_versioned_docs/version-24-1/get-started/upgrade-cluster.mdx new file mode 100644 index 0000000000..4504383550 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/get-started/upgrade-cluster.mdx @@ -0,0 +1,19 @@ +--- +description: Upgrade a connected Calico Cloud cluster to the latest released version of the in-cluster components. +--- + +# Upgrade Calico Cloud + +To upgrade a managed cluster to the latest version of $[prodname]: + +1. From the web console, go to the **Managed Clusters** page. +1. If your organization uses multiple projects to group managed clusters, click the **Project** menu and select the project you want your cluster to be part of. +1. For the cluster you want to upgrade, select **Actions** > **Reinstall**. +1. In the **Reinstall Cluster** dialog, select a newer version of $[prodname] from the list, click **Reinstall**, and copy the generated kubectl command. +1. From a terminal, paste and run the command. + The cluster's status under **Connection Status** changes to **Disconnected: Installing**. + When the upgrade is complete, the status changes to **Connected**. + + + + diff --git a/calico-cloud_versioned_docs/version-24-1/get-started/windows-limitations.mdx b/calico-cloud_versioned_docs/version-24-1/get-started/windows-limitations.mdx new file mode 100644 index 0000000000..bd2f0a15ad --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/get-started/windows-limitations.mdx @@ -0,0 +1,204 @@ +--- +description: Known limitations for Calico Cloud on Windows worker nodes that you should review before planning a connection. +--- + +# Limitations and known issues for Windows nodes + +## $[prodname] feature limitations + +| Feature | Unsupported in this release | +| ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Platforms | - GKE | +| Install and upgrade | - Typha component for scaling (Linux-based feature) | +| Networking | - Overlay mode with BGP peering
- IP in IP overlay with BGP routing
- Cross-subnet support and MTU setting for VXLAN
- IPv6 and dual stack
- Dual-ToR
- Service advertisement
- Multiple networks to pods | +| Policy | - Staged network-policy
- Firewall integrations
- Policy for hosts (host endpoints, including automatic host endpoints)
- Tiered policy: TKG, GKE, AKS
- WAF integration
- AWS firewall integration
- Fortinet integration | +| Visibility and troubleshooting | - Packet capture
- DNS logs
- iptables logs
- L7 logs | +| Threat defense | - No threat defense features are supported. | +| Multi-cluster management | - Multi-cluster management federated identity endpoints and services
- Federated endpoint identity and services | +| Security | - WireGuard encryption for pod-to-pod traffic and host-to-host traffic | +| Data plane | - eBPF is a Linux-based feature | + +## $[prodname] BGP networking limitations + +If you are using $[prodname] with BGP, note these current limitations with Windows. + +| Feature | Limitation | +| ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| IP mobility/ borrowing | $[prodname] IPAM allocates IPs to host in blocks for aggregation purposes.
If the IP pool is full, nodes can also "borrow" IPs from another node's block. In BGP terms, the borrower then advertises a more specific "/32" route for the borrowed IP and traffic for that IP is only routed to the borrowing host.

Windows nodes do not support this borrowing mechanism; they will not borrow IPs even if the IP pool is full and they mark their blocks so that Linux nodes will not borrow from them. | +| IPs reserved for Windows | $[prodname] IPAM allocates IPs in CIDR blocks. Due to networking requirements on Windows, four IPs per Windows node-owned block must be reserved for internal purposes.

For example, with the default block size of /26, each block contains 64 IP addresses, 4 are reserved for Windows, leaving 60 for pod networking.

To reduce the impact of these reservations, a larger block size can be configured at the IP pool scope (before any pods are created). | +| Single IP block per host | $[prodname] IPAM is designed to allocate blocks of IPs (default size /26) to hosts on demand. While the $[prodname] CNI plugin was written to do the same, kube-proxy for Windows currently only supports a single IP block per host.

To work around the default limit of one /26 per host there some options:

- Use $[prodname] BGP networking with the kubernetes datastore. In that mode, $[prodname] IPAM is not used and the CNI host-local IPAM plugin is used with the node's Pod CIDR.

To allow multiple IPAM blocks per host (at the expense of kube-proxy compatibility), set the `windows_use_single_network` flag to `false` in the `cni.conf.template` before installing $[prodname]. Changing that setting after pods are networked is not recommended because it may leak HNS endpoints. | +| IP-in-IP overlay | $[prodname]'s IPIP overlay mode cannot be used in clusters that contain Windows nodes because Windows does not support IP-in-IP. | +| NAT-outgoing | $[prodname] IP pools support a "NAT outgoing" setting with the following behaviour:

- Traffic between $[prodname] workloads (in any IP pools) is not NATted.
- Traffic leaving the configured IP pools is NATted if the workload has an IP within an IP pool that has NAT outgoing enabled. $[prodname] honors the above setting but it is only applied at pod creation time. If the IP pool configuration is updated after a pod is created, the pod's traffic will continue to be NATted (or not) as before. NAT policy for newly-networked pods will honor the new configuration. $[prodname] automatically adds the host itself and its subnet to the NAT exclusion list. This behaviour can be disabled by setting flag `windows_disable_host_subnet_nat_exclusion` to `true` in `cni.conf.template` before running the install script. | +| Service IP advertisement | This $[prodname] feature is not supported on Windows. | + +### Check your network configuration + +If you are using a networking type that requires layer 2 reachability (such as $[prodname] with a BGP mesh and no peering to your fabric), you can check that your network has layer 2 reachability as follows: + +On each of your nodes, check the IP network of the network adapter that you plan to use for pod networking. For example, on Linux, assuming your network adapter is eth0, you can run: + +``` +$ ip addr show eth0 + 2: eth0: mtu 1500 qdisc pfifo_fast state UP group default qlen 1000 + + link/ether 00:0c:29:cb:c8:19 brd ff:ff:ff:ff:ff:ff + inet 192.168.171.136/24 brd 192.168.171.255 scope + + global eth0 + valid_lft forever preferred_lft forever + inet6 fe80::20c:29ff:fecb:c819/64 scope + link + + valid_lft forever preferred_lft + forever +``` + +In this case, the IPv4 is 192.168.171.136/24; which, after applying the /24 mask gives 192.168.171.0/24 for the IP network. + +Similarly, on Windows, you can run + +``` +PS C:\> ipconfig + +Windows IP Configuration + +Ethernet adapter vEthernet (Ethernet 2): + + Connection-specific DNS Suffix . : + us-west-2.compute.internal Link-local IPv6 Address . . . . + . : fe80::6d10:ccdd:bfbe:bce2%15 IPv4 Address. . . . . . . + . . . . : 172.20.41.103 Subnet Mask . . . . . . . . . . . + : 255.255.224.0 Default Gateway . . . . . . . . . : + 172.20.32.1 + +``` + +In this case, the IPv4 address is 172.20.41.103 and the mask is represented as bytes 255.255.224.0 rather than CIDR notation. Applying the mask, we get a network address 172.20.32.0/19. + +Because the Linux node has network 192.168.171.136/24 and the Windows node has a different network, 172.20.32.0/19, they are unlikely to be on the same layer 2 network. + +## VXLAN networking limitations + +Because of differences between the Linux and Windows data plane feature sets, the following $[prodname] features are not supported on Windows. + +| Feature | Limitation | +| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| IPs reserved for Windows | $[prodname] IPAM allocates IPs in CIDR blocks. Due to networking requirements on Windows, four IPs per Windows node-owned block must be reserved for internal purposes.

For example, with the default block size of /26, each block contains 64 IP addresses, 4 are reserved for Windows, leaving 60 for pod networking.

To reduce the impact of these reservations, a larger block size can be configured at the IP pool scope (before any pods are created). | +| Single IP block per host | $[prodname] IPAM is designed to allocate blocks of IPs (default size /26) to hosts on demand. While the $[prodname] CNI plugin was written to do the same, kube-proxy currently only supports a single IP block per host.
To allow multiple IPAM blocks per host (at the expense of kube-proxy compatibility), set the `windows_use_single_network` flag to `false` in the `cni.conf.template` before installing $[prodname]. Changing that setting after pods are networked is not recommended because it may leak HNS endpoints. | + +## Routes are lost in cloud providers + +If you create a Windows host with a cloud provider (AWS for example), the creation of the vSwitch at $[prodname] install time can remove the cloud provider's metadata route. If your application relies on the metadata service, you may need to examine the routing table before and after installing $[prodname] to reinstate any lost routes. + +## VXLAN limitations + +**VXLAN support** + +- Windows 1903 build 18317 and above +- Windows 1809 build 17763 and above + +**Configuration updates** + +Certain configuration changes will not be honored after the first pod is networked. This is because Windows does not currently support updating the VXLAN subnet parameters after the network is created so updating those parameters requires the node to be drained: + +One example is the VXLAN VNI setting. To change such parameters: + +- Drain the node of all pods +- Delete the $[prodname] HNS network: + + ```powershell + Import-Module -DisableNameChecking $[rootDirWindows]\libs\hns\hns.psm1 + Get-HNSNetwork | ? Name -EQ "$[prodname]" | Remove-HNSNetwork + ``` + +- Update the configuration in `config.ps1`, run `uninstall-calico.ps1` and then `install-calico.ps1` to regenerate the CNI configuration. + +## Pod-to-pod connections are dropped with TCP reset packets + +Restarting Felix or changes to policy (including changes to endpoints referred to in policy) can cause pod-to-pod connections to be dropped with TCP reset packets when one of the following occurs: + +- The policy that applies to a pod is updated +- Some ingress or egress policy that applies to a pod contains selectors and the set of endpoints that those selectors match changes + +Felix must reprogram the HNS ACL policy attached to the pod. This reprogramming can cause TCP resets. Microsoft has confirmed this is a HNS issue, and they are investigating. + +## Service ClusterIPs incompatible with selectors on pod IPs in network policy + +**Windows 1809 prior to build 17763.1432** + +On Windows nodes, kube-proxy unconditionally applies source NAT to traffic from local pods to service ClusterIPs. This means that, at the destination pod, where policy is applied, the traffic appears to come from the source host rather than the source pod. In turn, this means that a network policy with a source selector matching the source pod will not match the expected traffic. + +## Network policy and using selectors + +Under certain conditions, relatively simple $[prodname] policies can require significant Windows data plane resources, that can cause significant CPU and memory usage, and large policy programming latency. + +We recommend avoiding policies that contain rules with both a source and destination selector. The following is an example of a policy that would be inefficient. The policy applies to all workloads, and it only allows traffic from workloads labeled as clients to workloads labeled as servers: + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: calico-dest-selector +spec: + selector: all() + order: 500 + ingress: + - action: Allow + destination: + selector: role == "webserver" + source: + selector: role == "client" +``` + +Because the policy applies to all workloads, it will be rendered once per workload (even if the workload is not labeled as a server), and then the selectors will be expanded into many individual data plane rules to capture the allowed connectivity. + +Here is a much more efficient policy that still allows the same traffic: + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: calico-dest-selector +spec: + selector: role == "webserver" + order: 500 + ingress: + - action: Allow + source: + selector: role == "client" +``` + +The destination selector is moved into the policy selector, so this policy is only rendered for workloads that have the `role: webserver` label. In addition, the rule is simplified so that it only matches on the source of the traffic. Depending on the number of webserver pods, this change can reduce the data plane resource usage by several orders of magnitude. + +## Network policy with tiers + +Because of the way the Windows data plane handles rules, the following limitations are required to avoid performance issues: + +- Tiers: maximum of 5 +- `pass` rules: maximum of 10 per tier +- If each tier contains a large number of rules, and has pass rules, you may need to reduce the number of tiers further. + +## Flow log limitations + +$[prodname] supports flow logs with these limitations: + +- No packet/bytes stats for denied traffics +- Inaccurate `num_flows_started` and `num_flows_completed` stats with VXLAN networking +- No DNS stats +- No HTTP stats +- No RuleTrace for tiers +- No BGP logs + +## DNS Policy limitations + +:::note + +DNS Policy is a tech preview feature. Tech preview features may be subject to significant changes before they become GA. + +::: + +$[prodname] supports DNS policy on Windows with these limitations: + +- It could take up to 5 seconds for the first TCP SYN packet to go through, for a connection to a DNS domain name. This is because DNS policies are dynamically programmed. The first TCP packet could be dropped since there is no policy to allow it until $[prodname] detects domain IPs from DNS response and programs DNS policy rules. The Windows TCPIP stack will send SYN again after TCP Retransmission timeout (RTO) if previous SYN has been dropped. +- Some runtime libraries do not honour DNS TTL. Instead, they manage their own DNS cache which has a different TTL value for DNS entries. On .NET Framework, the value to control DNS TTL is ServicePointManager.DnsRefreshTimeout which has default value of 120 seconds - [DNS refresh timeout](https://docs.microsoft.com/en-us/dotnet/api/system.net.servicepointmanager.dnsrefreshtimeout). It is important that $[prodname] uses a longer TTL value than the one used by the application, so that DNS policy will be in place when the application is making outbound connections. The configuration item “WindowsDNSExtraTTL” should have a value bigger than the maximum value of DNS TTL used by the runtime libraries for your applications. +- Due to the limitations of Windows container networking, a policy update could have an impact on performance. Programming DNS policy may result in more policy updates. Setting “WindowsDNSExtraTTL” to a bigger number will reduce the performance impact. diff --git a/calico-cloud_versioned_docs/version-24-1/multicluster/aws.mdx b/calico-cloud_versioned_docs/version-24-1/multicluster/aws.mdx new file mode 100644 index 0000000000..bdfaf19a16 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/multicluster/aws.mdx @@ -0,0 +1,68 @@ +--- +description: Sample Calico Cloud cluster mesh configuration that peers an on-premises cluster with an AWS VPC cluster using BGP, NLB, and the Federated Services Controller. +--- + +# Cluster mesh example for clusters in AWS + +## Big picture + +A sample configuration for cluster mesh using AWS clusters. + +## Tutorial + +**Set up** + +The cluster is installed on real hardware where node and pod IPs are routable, using an edge VPN router to peer with the AWS cluster. + +![A diagram showing the key configuration requirements setting up an AWS cluster (using AWS VPN CNI) peering with an on-premise cluster.](/img/calico-enterprise/federation/aws-rcc.svg) + +**Calico Cloud configuration** + +- IP pool resource is configured for the on-premise IP assignment with IPIP is disabled +- BGP peering to the VPN router +- A Remote Cluster Configuration resource references the AWS cluster +- Service discovery of the AWS cluster services uses the Calico Cloud Federated Services Controller + +**Notes** + +- If VPN Router is configured as a route reflector for the on-premise cluster, you would: + - Configure the default BGP Configuration resource to disable node-to-node mesh + - Configure a global BGP Peer resource to peer with the VPN router +- If the IP Pool has `Outgoing NAT` enabled, then you must add an IP Pool covering the AWS cluster VPC with disabled set to `true`. When set to `true` the pool is not used for IP allocations, and SNAT is not performed for traffic to the AWS cluster. + +**AWS configuration** + +- A VPC CIDR is chosen that does not overlap with the on-premise IP ranges. +- There are 4 subnets within the VPC, split across two AZs (for availability) such that each AZ has a public and private subnet. In this particular example, the split of responsibility is: + - The private subnet is used for node and pod IP allocation + - The public subnet is used to home a NAT gateway for pod-to-internet traffic. +- The VPC is peered to an on-premise network using a VPN. This is configured as a VPN gateway for the AWS side, and a classic VPN for the customer side. BGP is used for route distribution. +- Routing table for private subnet has: + - "propagate" set to "true" to ensure BGP-learned routes are distributed + - Default route to the NAT gateway for public internet traffic + - Local VPC traffic +- Routing table for public subnet has default route to the internet gateway. +- Security group for the worker nodes has: + - Rule to allow traffic from the peered networks + - Other rules required for settings up VPN peering (refer to the AWS docs for details). + +To automatically create a Network Load Balancer (NLB) for the AWS deployment, we apply a service with the correct annotation. + +```yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + service.beta.kubernetes.io/aws-load-balancer-type: nlb + name: nginx-external +spec: + externalTrafficPolicy: Local + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 80 + selector: + run: nginx + type: LoadBalancer +``` diff --git a/calico-cloud_versioned_docs/version-24-1/multicluster/index.mdx b/calico-cloud_versioned_docs/version-24-1/multicluster/index.mdx new file mode 100644 index 0000000000..58be3bb57d --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/multicluster/index.mdx @@ -0,0 +1,17 @@ +--- +description: Form a Calico Cloud cluster mesh across clusters connected to the SaaS management plane with identity-aware policy and cross-cluster service discovery. +hide_table_of_contents: true +--- + +import { DocCardLink, DocCardLinkLayout } from '/src/___new___/components'; + +# Cluster mesh + +With cluster mesh, you can secure cross-cluster connections using identity-aware network policy and federate services for cross-cluster service discovery. + + + + + + + \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/multicluster/kubeconfig.mdx b/calico-cloud_versioned_docs/version-24-1/multicluster/kubeconfig.mdx new file mode 100644 index 0000000000..94d26b27d7 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/multicluster/kubeconfig.mdx @@ -0,0 +1,404 @@ +--- +description: Connect Calico Cloud connected clusters into a cluster mesh by generating per-cluster kubeconfig credentials and RemoteClusterConfiguration with VXLAN or WireGuard. +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Creating the cluster mesh +In this page, we will create a $[prodname] cluster mesh by connecting clusters together. Once created, $[prodname] cluster mesh enables multi-cluster networking, network policy for cross-cluster connections, cross-cluster services, and encryption via WireGuard. + +## Requirements +$[prodname] multi-cluster networking provides routing between clusters that preserves pod IPs. This section outlines the requirements for this routing to be established. If your network already provides routing between clusters that preserves pod IPs, you can skip this section. + +### Prerequisites for $[prodname] multi-cluster networking: +- All nodes participating in the cluster mesh must be able to establish connections to each other via their private IP. +- All nodes participating in the cluster mesh must have unique node names. +- Pod CIDRs between clusters must not overlap. +- All clusters must have at least one overlay network in common (VXLAN and/or WireGuard). +- All clusters must have the same `routeSource` setting on `FelixConfiguration`. + +If using VXLAN: +- The `vxlan*` settings on `FelixConfiguration` must be the same across clusters participating in the mesh. +- The underlying network must allow traffic on `vxlanPort` between clusters participating in the mesh. +- All clusters must use Calico CNI. + +If using WireGuard: +- The `wireguard*` settings on `FelixConfiguration` must be the same across clusters participating in the mesh. +- The underlying network must allow traffic on `wireguardListeningPort` between clusters participating in the mesh. +- All clusters must use Calico CNI OR All clusters must use non-Calico CNI (mixing non-Calico CNI types is supported). + +Note: much like intra-cluster routing in $[prodname], cross-cluster routing can utilize both VXLAN and WireGuard at the same time. If both are enabled and a WireGuard peer is not ready, communication with that peer will fall back to VXLAN. + +## Setup + +### Generate credentials for cross-cluster resource synchronization +:::tip[mental model] +The basis of cluster mesh is the ability for a cluster to connect to a remote cluster and sync data from it. This enables each $[prodname] cluster to have a view into the datastore that includes both local and remote cluster pods. +::: + +In this section, we will create a `kubeconfig` for each cluster. This `kubeconfig` is what other clusters will use to connect to a given cluster and synchronize data from it. + +**For each** cluster in the cluster mesh, utilizing an existing `kubeconfig` with administrative privileges, follow these steps: + +1. Create the ServiceAccount used by remote clusters for authentication: + + ```bash + kubectl apply -f $[filesUrl_CE]/manifests/federation-remote-sa.yaml + ``` + +1. Create the ClusterRole and ClusterRoleBinding used by remote clusters for authorization: + + ```bash + kubectl apply -f $[filesUrl_CE]/manifests/federation-rem-rbac-kdd.yaml + ``` +1. Create the ServiceAccount token that will be used in the `kubeconfig`: + + ```yaml + kubectl apply -f - < $KUBECONFIG_NAME + ``` + +1. Verify that the `kubeconfig` file works: + + Issue the following command to validate the `kubeconfig` file can be used to connect to the current cluster and access resources: + ```bash + kubectl --kubeconfig=$KUBECONFIG_NAME get nodes + ``` + +Once you've created a `kubeconfig` for **each** cluster, you can proceed to the next section to establish the cluster connections that form the mesh. + +### Establish cross-cluster resource synchronization +:::tip[mental model] +The cluster mesh is formed when each cluster connects to every other cluster to synchronize data. A cluster connects to another cluster using a RemoteClusterConfiguration, which references a kubeconfig created for the remote cluster. +::: + +In this section, within each cluster, we will create a RemoteClusterConfiguration for each other cluster in the mesh. This RemoteClusterConfiguration instructs the cluster to connect to a cluster using a kubeconfig. + +With each cluster being connected to each other cluster, a full cluster mesh will be formed. + + + + +:::tip[mental model] +$[prodname] achieves cross-cluster routing by extending the overlay network of a cluster to include nodes from remote clusters. This is made possible by each cluster having a view into the datastore that now includes remote pods and nodes. +::: +**For each pair** of clusters in the cluster mesh (e.g. \{A,B\}, \{A,C\}, \{B,C\} for clusters A,B,C): + +1. In cluster 1, create a secret that contains the `kubeconfig` for cluster 2: + + Determine the namespace (``) for the secret to replace in all steps. + The simplest method to create a secret for a remote cluster is to use the `kubectl` command because it correctly encodes the data and formats the file. + ```bash + kubectl create secret generic remote-cluster-secret-name -n \ + --from-literal=datastoreType=kubernetes \ + --from-file=kubeconfig= + ``` + +1. If RBAC is enabled in cluster 1, create a Role and RoleBinding for $[prodname] to use to access the secret that contains the `kubeconfig` for cluster 2: + ```bash + kubectl create -f - < + rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["watch", "list", "get"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: remote-cluster-secret-access + namespace: + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: remote-cluster-secret-access + subjects: + - kind: ServiceAccount + name: calico-typha + namespace: calico-system + EOF + ``` + +1. Create the RemoteClusterConfiguration in cluster 1: + + Within the RemoteClusterConfiguration, we specify the secret used to access cluster 2, and the overlay routing mode which toggles the establishment of cross-cluster overlay routes. + ```bash + kubectl create -f - < + kind: Secret + syncOptions: + overlayRoutingMode: Enabled + EOF + ``` + +1. [Validate](#check-remote-cluster-connection) the that the remote cluster connection can be established. + +1. Repeat the above steps, switching cluster 1 and cluster 2. + + + +In this setup, the cluster mesh will rely on the underlying network to provides cross-cluster routing that preserves pod IPs. + +**For each pair** of clusters in the cluster mesh (e.g. \{A,B\}, \{A,C\}, \{B,C\} for clusters A,B,C): + +1. In cluster 1, create a secret that contains the `kubeconfig` for cluster 2: + + Determine the namespace (``) for the secret to replace in all steps. + The simplest method to create a secret for a remote cluster is to use the `kubectl` command because it correctly encodes the data and formats the file. + ```bash + kubectl create secret generic remote-cluster-secret-name -n \ + --from-literal=datastoreType=kubernetes \ + --from-file=kubeconfig= + ``` + +1. If RBAC is enabled in cluster 1, create a Role and RoleBinding for $[prodname] to use to access the secret that contains the `kubeconfig` for cluster 2: + ```bash + kubectl create -f - < + rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["watch", "list", "get"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: remote-cluster-secret-access + namespace: + roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: remote-cluster-secret-access + subjects: + - kind: ServiceAccount + name: calico-typha + namespace: calico-system + EOF + ``` + +1. Create the RemoteClusterConfiguration in cluster 1: + + Within the RemoteClusterConfiguration, we specify the secret used to access cluster 2, and the overlay routing mode which toggles the establishment of cross-cluster overlay routes. + ```bash + kubectl create -f - < + kind: Secret + syncOptions: + overlayRoutingMode: Disabled + EOF + ``` + +1. If you have no IP pools in cluster 1 with NAT-outgoing enabled, skip this step. + + Otherwise, if you have IP pools in cluster 1 with NAT-outgoing enabled, and workloads in that pool will egress to workloads in cluster 2, you need to instruct $[prodname] to not perform NAT on traffic destined for IP pools in cluster 2. + + You can achieve this by creating a disabled IP pool in cluster 1 for each CIDR in cluster 2. This IP pool should have NAT-outgoing disabled. For example: + + ```yaml + apiVersion: projectcalico.org/v3 + kind: IPPool + metadata: + name: cluster2-main-pool + spec: + cidr: + disabled: true + ``` + +1. [Validate](#check-remote-cluster-connection) the that the remote cluster connection can be established. + +1. Repeat the above steps, switching cluster 1 and cluster 2. + + + + +#### 🎉 Done! +After completing the above steps for all cluster pairs in the cluster mesh, your clusters should now be forming a cluster mesh! You should now be able to route traffic between clusters, and write policy that can select remote workloads. + +:::tip[mental model] +A cluster in the mesh can write policy rules that select pods from other clusters in the mesh. This is because traffic from remote clusters has pod IPs preserved, and the local cluster can associate remote pod IPs with the pod specs it synchronized from remote clusters. +::: + +## How to + +### Switch to multi-cluster networking +The steps above assume that you are configuring both federated endpoint identity and multi-cluster networking for the first time. If you already have federated endpoint identity, and want to use multi-cluster networking, follow these steps: + +1. Validate that all [requirements](#prerequisites-for-calico-cloud-multi-cluster-networking) for multi-cluster networking have been met. +2. Update the ClusterRole in each cluster in the cluster mesh using the RBAC manifest found in [Generate credentials for cross-cluster authentication](#generate-credentials-for-cross-cluster-resource-synchronization) +3. In all RemoteClusterConfigurations, set `Spec.OverlayRoutingMode` to `Enabled`. +4. Verify that all RemoteClusterConfigurations are bidirectional (in both directions for each cluster pair) using these [instructions](#establish-cross-cluster-resource-synchronization). +5. If you had previously created disabled IP pools to prevent NAT outgoing from applying to remote cluster destinations, those disabled IP pools are no longer needed when using multi-cluster networking and must be deleted. + +### Validate federated endpoint identity & multi-cluster networking +#### Validate RemoteClusterConfiguration and federated endpoint identity +##### Check remote cluster connection + +You can check the Typha logs for remote cluster connection status. Run the following command: + +```bash +kubectl logs deployment/calico-typha -n calico-system | grep "Sending in-sync update" +``` +You should see an entry for each RemoteClusterConfiguration in the local cluster. + +If either output contains unexpected results, proceed to the [troubleshooting](#troubleshoot) section. + +#### Validate multi-cluster networking +If all requirements were met for $[prodname] to establish multi-cluster networking, you can test the functionality by establishing a connection from a pod in a local cluster to the IP of a pod in a remote cluster. Ensure that there is no policy in either cluster that may block this connection. + +If the connection fails, proceed to the [troubleshooting](#troubleshoot) section. + +### Create remote-identity-aware network policy +With federated endpoint identity and routing between clusters established, you can now use labels to reference endpoints on a remote cluster in local policy rules, rather than referencing them by IP address. + +The main policy selector still refers only to local endpoints; and that selector chooses which local endpoints to apply the policy. +However, rule selectors can now refer to both local and remote endpoints. + +In the following example, cluster A (an application cluster) has a network policy that governs outbound connections to cluster B (a database cluster). +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: default.app-to-db + namespace: myapp +spec: + # The main policy selector selects endpoints from the local cluster only. + selector: app == 'backend-app' + tier: default + egress: + - destination: + # Rule selectors can select endpoints from local AND remote clusters. + selector: app == 'postgres' + protocol: TCP + ports: [5432] + action: Allow +``` + +### Troubleshoot +#### Troubleshoot RemoteClusterConfiguration and federated endpoint identity + +##### Verify configuration +For each impacted remote cluster pair (between cluster A and cluster B): +1. Retrieve the `kubeconfig` from the secret stored in cluster A. Manually verify that it can be used to connect to cluster B. + ```bash + kubectl get secret -n remote-cluster-secret-name -o=jsonpath="{.data.kubeconfig}" | base64 -d > verify_kubeconfig_b + kubectl --kubeconfig=verify_kubeconfig_b get nodes + ``` + This validates that the credentials used by Typha to connect to cluster B's API server are stored in the correct location and provide sufficient access. + + The command above should yield a result like the following: + ``` + NAME STATUS ROLES AGE VERSION + clusterB-master Ready master 7d v1.27.0 + clusterB-worker-1 Ready worker 7d v1.27.0 + clusterB-worker-2 Ready worker 7d v1.27.0 + ``` + + If you do not see the nodes of cluster B listed in response to the command above, verify that you [created](#generate-credentials-for-cross-cluster-resource-synchronization) the `kubeconfig` for cluster B correctly, and that you [stored](#establish-cross-cluster-resource-synchronization) it in cluster A correctly. + + If you do see the nodes of cluster B listed in response to the command above, you can run this test (or a similar test) on a node in cluster A to verify that cluster A nodes can connect to the API server of cluster B. + +2. Validate that the Typha service account in Cluster A is authorized to retrieve the `kubeconfig` secret for cluster B. + ```bash + kubectl auth can-i list secrets --namespace --as=system:serviceaccount:calico-system:calico-typha + ``` + + This command should yield the following output: + ``` + yes + ``` + + If the command does not return this output, verify that you correctly [configured RBAC](#establish-cross-cluster-resource-synchronization) in cluster A. + +3. Repeat the above, switching cluster A to cluster B. + +##### Check logs +Validate that querying Typha logs yield the expected result outlined in the [validation](#validate-federated-endpoint-identity--multi-cluster-networking) section. + +If the Typha logs do not yield the expected result, review the warning or error-related logs in `typha` or `calico-node` for insights. + +#### Troubleshoot multi-cluster networking +##### Basic validation +* Ensure that RemoteClusterConfiguration and federated endpoint identity are [functioning correctly](#validate-federated-endpoint-identity--multi-cluster-networking) +* Verify that you have met the [prerequisites](#prerequisites-for-calico-cloud-multi-cluster-networking) for multi-cluster networking +* If you had previously set up RemoteClusterConfigurations without multi-cluster networking, and are upgrading to use the feature, review the [switching considerations](#switch-to-multi-cluster-networking) +* Verify that traffic between clusters is not being denied by network policy + +##### Check overlayRoutingMode +Ensure that `overlayRoutingMode` is set to `"Enabled"` on all RemoteClusterConfigurations. + +If overlay routing is successfully enabled, you can view the logs of a Typha instance using: +```bash +kubectl logs deployment/calico-typha -n calico-system +``` + +You should see an output for each connected remote cluster that looks like this: +``` +18:49:35.394 [INFO][14] wrappedcallbacks.go 443: Creating syncer for RemoteClusterConfiguration(my-cluster) +18:49:35.394 [INFO][14] watchercache.go 186: Full resync is required ListRoot="/calico/ipam/v2/assignment/" +18:49:35.395 [INFO][14] watchercache.go 186: Full resync is required ListRoot="/calico/resources/v3/projectcalico.org/workloadendpoints" +18:49:35.396 [INFO][14] watchercache.go 186: Full resync is required ListRoot="/calico/resources/v3/projectcalico.org/hostendpoints" +18:49:35.396 [INFO][14] watchercache.go 186: Full resync is required ListRoot="/calico/resources/v3/projectcalico.org/profiles" +18:49:35.396 [INFO][14] watchercache.go 186: Full resync is required ListRoot="/calico/resources/v3/projectcalico.org/nodes" +18:49:35.397 [INFO][14] watchercache.go 186: Full resync is required ListRoot="/calico/resources/v3/projectcalico.org/ippools" +``` + +If you do not see the each of the resource types above, overlay routing was not successfully enabled in your cluster. Verify that you followed the [setup](#establish-cross-cluster-resource-synchronization) correctly for overlay routing, and that the cluster is using a version of $[prodname] that supports multi-cluster networking. + +###### Check logs +Warning or error logs in `typha` or `calico-node` may provide insight into where issues are occurring. + +## Next steps + +[Configure federated services](services-controller.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/multicluster/overview.mdx b/calico-cloud_versioned_docs/version-24-1/multicluster/overview.mdx new file mode 100644 index 0000000000..a93d613050 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/multicluster/overview.mdx @@ -0,0 +1,54 @@ +--- +description: Calico Cloud cluster mesh concepts for connected clusters including pod IP routability, federated endpoint identity, federated services, and overlay networking. +--- + +# Overview + +## Big picture + +Secure cross-cluster connections with identity-aware network policy, and federate services for cross-cluster service discovery. + +Utilize $[prodname] to establish cross-cluster connectivity. + +## Value + +At some point in your Kubernetes journey, you may have applications that need to access services and workloads running in another cluster. + +By default, pods can only communicate with pods within the same cluster. Additionally, services and network policy only select pods from within the same cluster. $[prodname] can help overcome these barriers by forming a cluster mesh that provides the following features: +- **Federated endpoint identity** + + Allow a local Kubernetes cluster to include the workload endpoints (pods) and host endpoints of a remote cluster in the calculation of local network policies applied on each node of the local cluster. + +- **Federated services** + + Enable a local Kubernetes Service to populate with Endpoints selected from both local cluster and remote cluster Services. + +- **Multi-cluster networking** + + Establish an overlay network between clusters to provide cross-cluster connectivity with $[prodname]. + +## Concepts + +### Pod IP routability + +$[prodname] cluster mesh is implemented at Kubernetes at the network layer, based on pod IPs. + +Taking advantage of federated workload endpoint identity and federated services requires that pod IPs are routable between clusters. This is because identity-aware network policy requires source and destination pod IPs to be preserved to establish pod identity. Additionally, the Endpoint IPs of pods selected by a federated Service must be routable in order for that Service to be of value. + +You can utilize $[prodname] multi-cluster networking to establish pod IP routability between clusters via overlay. Alternatively, you can manually set up pod IP routability between clusters without encapsulation (e.g. VPC routing, BGP routing). + +### Federated endpoint identity + +Federated endpoint identity in a cluster mesh allows a local Kubernetes cluster to include the workload endpoints (pods) and host endpoints of a remote cluster in the calculation of the local policies for each node, e.g. Cluster A network policy allows its application pods to talk to database pods in Cluster B. + +This feature does not _federate network policies_; policies from a remote cluster are not applied to the endpoints on the local cluster, and the policy from the local cluster is rendered only locally and applied to the local endpoints. + +### Federated services + +Federated services in a cluster mesh works with federated endpoint identity, providing cross-cluster service discovery for a local cluster. If you have an existing service discovery mechanism, this feature is optional. + +Federated services use the Tigera Federated Services Controller to federate all Kubernetes endpoints (workload and host endpoints) across all of the clusters. The Federated Services Controller accesses service and endpoints data in the remote clusters directly through the Kubernetes API. + +## Next steps + +[Configure remote-aware policy and multi-cluster networking](kubeconfig.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/multicluster/services-controller.mdx b/calico-cloud_versioned_docs/version-24-1/multicluster/services-controller.mdx new file mode 100644 index 0000000000..2c8435578d --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/multicluster/services-controller.mdx @@ -0,0 +1,209 @@ +--- +description: Configure a Calico Cloud federated service with the Tigera Federated Services Controller to surface endpoints from connected clusters in the SaaS plane. +--- + +# Configure federated services + +## Big picture + +Configure local clusters to discover services across multiple clusters. + +## Value + +Use cluster mesh and federated services discovery along with federated endpoint identity to extend and automate endpoints sharing. (Optional if you have your own service discovery mechanism.) + +## Concepts + +### Federated services + +A federated service (also called a backing service), is a set of services with consolidated endpoints. $[prodname] discovers services across a cluster mesh (both local cluster and remote clusters) and creates a "federated service" on the local cluster that encompasses all of the individual services. + +Federated services are managed by the Tigera Federated Service Controller, which monitors and maintains endpoints for each locally-federated service. The controller does not change configuration on remote clusters. + +A federated service looks similar to a regular Kubernetes service, but instead of using a pod selector, it uses an annotation. For example: + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: my-app-federated + namespace: default + annotations: + federation.tigera.io/serviceSelector: run == "my-app" +spec: + ports: + - name: my-app-ui + port: 8080 + protocol: TCP + type: ClusterIP +``` + +| Annotation | Description | +| -------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `federation.tigera.io/serviceSelector` | Required field that specifies the services used in the federated service. Format is a standard $[prodname] selector (i.e. the same as $[prodname] policy resources) and selects services based on their labels. The selector annotation selects services, not pods.

Only services in the same namespace as the federated service are included. This implies namespace names across clusters are linked (this is a basic premise of federated endpoint identity).

If the value is incorrectly specified, the service is not federated and endpoint data is removed from the service. View the warning logs in the controller for any issues processing this value. | + +**Syntax and rules** + +- Services that you specify in the federated service must be in the same namespace or they are ignored. A basic assumption of federated endpoint identity is that namespace names are linked across clusters. +- If you specify a `spec.Selector` in a federated service, the service is not federated. +- You cannot federate another federated service. If a service has a federated services annotation, it is not included as a backing service of another federated service. +- The target port number in the federated service ports is not used. + +**Match services using a label** + +You can also match services using a label. The label is implicitly added to each service, but it does not appear in `kubectl` when viewing the service. + +| Label | Description | +| ---------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `federation.tigera.io/remoteClusterName` | Label added to all remote services that correspond to the Remote Cluster Configuration name for the remote cluster. Use this label to restrict the clusters selected by the services. **Note**: The label is not added for services in the local cluster. | + +**About endpoints** + +- Do not manually create or manage endpoints resources; let the Tigera controller do all of the work. User updates to endpoint resources are ignored. +- Endpoints are selected only when the service port name and protocol in the federated service matches the port name and protocol in the backing service. +- Endpoint data configured in the federated service is slightly modified from the original data of the backing service. For backing services on remote clusters, the `targetRef.name` field in the federated service adds the ``. For example, `/`. + +## Before you begin + +**Required** + +- [Configure federated endpoint identity](kubeconfig.mdx) + +## How to + +- [Create service resources](#create-service-resources) +- [Create a federated service](#create-a-federated-service) +- [Access a federated service](#access-a-federated-service) + +### Create service resources + +On each cluster in the mesh that is providing a particular service, create your service resources as you normal would with the following requirements: + +- Services must all be in the same namespace. +- Configure each service with a common label key and value to identify the common set of services across your clusters (for example, `run=my-app`). + +Kubernetes manages the service by populating the service endpoints from the pods that match the selector configured in the service spec. + +### Configure a federated service + +1. On a cluster that needs to access the federated set of pods that are running an application, create a + service on that cluster leaving the `spec selector` blank. +1. Set the `federation.tigera.io/serviceSelector` annotation to be a $[prodname] selector that selects the previously-configured services using the matching label match (for example, `run == "my-app"`). + +The Federated Services Controller manages this service, populating the service endpoints from all of the services that match the service selector configured in the annotation. + +### Access a federated service + +Any application can access the federated service using the local DNS name for that service. The simplest way to access a federated service is through its corresponding DNS name. + +By default, Kubernetes adds DNS entries to access a service locally. For a service called `my-svc` in the namespace +`my-namespace`, the following DNS entry would be added to access the service within the local cluster: + +``` +my-svc.my-namespace.svc.cluster.local +``` + +DNS lookup for this name returns the fixed ClusterIP address assigned for the federated service. The ClusterIP is translated in iptables to one of the federated service endpoint IPs, and is load balanced across all of the endpoints. + +## Tutorial + +### Create a service + +In the following example, the remote cluster defines the following service. + +```yaml +apiVersion: v1 +kind: Service +metadata: + labels: + run: my-app + name: my-app + namespace: default +spec: + selector: + run: my-app + ports: + - name: my-app-ui + port: 80 + protocol: TCP + targetPort: 9000 + - name: my-app-console + port: 81 + protocol: TCP + targetPort: 9001 + type: ClusterIP +``` + +This service definition exposes two ports for the application `my-app`. One port for accessing a UI, and the other for accessing a management console. The service specifies a Kubernetes selector, which means the endpoints for this service are automatically populated by Kubernetes from matching pods within the services own cluster. + +### Create a federated service + +To create a federated service on your local cluster that federates the web access port for both the local and remote service, you would create a service resource on your local cluster as follows: + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: my-app-federated + namespace: default + annotations: + federation.tigera.io/serviceSelector: run == "my-app" +spec: + ports: + - name: my-app-ui + port: 8080 + protocol: TCP + type: ClusterIP +``` + +The `spec.selector` is not specified so it will not be managed by Kubernetes. Instead, we use a `federation.tigera.io/selector` annotation, indicating that this is a federated service managed by the Federated Services Controller. + +The controller matches the `my-app` services (matching the run label) on both the local and remote clusters, and consolidates endpoints from the `my-app-ui` TCP port for both of those services. Because the federated service does not specify the `my-app-console` port, the controller does not include these endpoints in the federated service. + +The endpoints data for the federated service is similar to the following. Note that the name of the remote cluster is included in `targetRef.name`. + +```yaml +apiVersion: v1 +kind: Endpoints +metadata: + creationTimestamp: 2018-07-03T19:41:38Z + annotations: + federation.tigera.io/serviceSelector: run == "my-app" + name: my-app-federated + namespace: default + resourceVersion: '701812' + selfLink: /api/v1/namespaces/default/endpoints/my-app-federated + uid: 1a0427e8-7ef9-11e8-a24c-0259d75c6290 +subsets: + - addresses: + - ip: 192.168.93.12 + nodeName: node1.localcluster.tigera.io + targetRef: + kind: Pod + name: my-app-59cf48cdc7-frf2t + namespace: default + resourceVersion: '701655' + uid: 19f5e914-7ef9-11e8-a24c-0259d75c6290 + ports: + - name: my-app-ui + port: 80 + protocol: TCP + - addresses: + - ip: 192.168.0.28 + nodeName: node1.remotecluster.tigera.io + targetRef: + kind: Pod + name: remotecluster/my-app-7b6f758bd5-ctgbh + namespace: default + resourceVersion: '701648' + uid: 19e2c841-7ef9-11e8-a24c-0259d75c6290 + ports: + - name: my-app-ui + port: 80 + protocol: TCP +``` + +## Additional resources + +- [Cluster mesh example for AWS](aws.mdx) +- [Federated services controller](../reference/resources/kubecontrollersconfig.mdx#federatedservicescontroller) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/application-layer-policies/alp-tutorial.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/application-layer-policies/alp-tutorial.mdx new file mode 100644 index 0000000000..4a1bce5fb7 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/application-layer-policies/alp-tutorial.mdx @@ -0,0 +1,168 @@ +--- +description: Step-by-step tutorial for applying Calico Cloud application-layer policy to workloads in a connected cluster — control ingress traffic by HTTP attributes. +--- + +# Application layer policy tutorial + +This tutorial shows how to use $[prodname] application layer policy to restrict ingress traffic for applications and microservices. + +### Install the demo application + +We will use a simple microservice application to demonstrate $[prodname] +application layer policy. The [YAO Bank](https://github.com/projectcalico/yaobank) application creates a +customer-facing web application, a microservice that serves up account +summaries, and an [etcd](https://github.com/coreos/etcd) datastore. + +```bash +kubectl apply -f $[tutorialFilesURL]/10-yaobank.yaml +``` +```bash +namespace/yaobank configured +service/database created +serviceaccount/database created +deployment.apps/database created +service/summary created +serviceaccount/summary created +deployment.apps/summary created +service/customer created +serviceaccount/customer created +deployment.apps/customer created +``` + +:::note + +You can also +[view the manifest in your browser](/files/10-yaobank.yaml). + +::: + +Verify that the application pods have been created and are ready. +```bash + kubectl rollout status deploy/summary deploy/customer deploy/database +``` + +When the demo application is displayed, you will see three pods. + +``` +NAME READY STATUS RESTARTS AGE +customer-2809159614-qqfnx 3/3 Running 0 21h +database-1601951801-m4w70 3/3 Running 0 21h +summary-2817688950-g1b3n 3/3 Running 0 21h +``` + +## Set up +- A $[prodname] cluster is running with application layer policy enabled +- Cluster has three microservices: customer, database, summary +- The customer web service should not have access to the backend database, but should have access to clients outside the cluster + +Imagine what would happen if an attacker were to gain control of the customer web pod in our +application. Let's simulate this by executing a remote shell inside that pod. + +```bash +kubectl exec -ti customer- -c customer -- bash +``` + +Notice that from here, we get direct access to the backend database. For example, we can list all the entries in the database like this: + +```bash +curl http://database:2379/v2/keys?recursive=true | python -m json.tool +``` + +(Piping to `python -m json.tool` nicely formats the output.) + +## Apply application layer policy + +In this step, we get the application layer policy YAML and apply it. Note that the policy scope is cluster-wide. + +With a $[prodname] policy, you can mitigate risks to the banking application. + +```bash +wget $[tutorialFilesURL]/30-policy.yaml +kubectl create -f 30-policy.yaml +``` + +Let's examine this policy piece by piece. First, notice that an application layer policy looks like a regular $[prodname] global network policy. +The only difference you'll see is the ability to use the application layer policy parameters in global network policy. Another important difference +is that you'll see HTTP traffic flows in the web console in features like Service Graph. + +Next, there are three policy objects, one for each microservice. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: customer +spec: + selector: app == 'customer' + ingress: + - action: Allow + http: + methods: ['GET'] + egress: + - action: Allow +``` + +The first policy protects the customer web app. Because this application is customer-facing, we do not +restrict what can communicate with it. We do, however, restrict its communications to HTTP `GET` +requests. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: summary +spec: + selector: app == 'summary' + ingress: + - action: Allow + source: + serviceAccounts: + names: ['customer'] + egress: + - action: Allow +``` + +The second policy protects the account summary microservice. We know the only consumer of this +service is the customer web app, so we restrict the source of incoming connections to the service +account for the customer web app. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: database +spec: + selector: app == 'database' + ingress: + - action: Allow + source: + serviceAccounts: + names: ["summary"] + egress: + - action: Allow +``` + +The third policy protects the database. Only the summary microservice should have direct access to +the database. + +### Verify the policy is working + +Let's verify our policy is working as intended. First, return to your browser and refresh to +ensure policy enforcement has not broken the application. + +Next, return to the customer web app. Recall that we simulated an attacker gaining control of that +pod by executing a remote shell inside it. + +```bash +kubectl exec -ti customer- -c customer bash +``` + +Repeat our attempt to access the database. + +```bash +curl -I http://database:2379/v2/keys?recursive=true +``` + +We omitted the JSON formatting because we do not expect to get a valid JSON response. This +time we should get a `403 Forbidden` response. Only the account summary microservice has database +access according to our policy. diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/application-layer-policies/alp.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/application-layer-policies/alp.mdx new file mode 100644 index 0000000000..6955c3d05c --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/application-layer-policies/alp.mdx @@ -0,0 +1,96 @@ +--- +description: Configure access controls based on Layer-7 attributes by enforcing Calico Cloud application-layer policy in connected clusters. +--- + +# Enable and enforce application layer policies + +:::warning[deprecation and removal notice] + +Application layer policy based on Envoy is deprecated and will be removed in an upcoming release. +For similar functionality, try [Istio Ambient Mode](../../compliance/istio/about-istio-ambient.mdx) with [Istio authorization policies](https://istio.io/latest/docs/ambient/getting-started/enforce-auth-policies/). + +::: + +Application layer policies let you configure access controls based on L7 attributes. + +## Before you begin + +### Limitations + +* Application layer policy is not compatible with a service mesh such as Istio. +* Application layer policy can restrict only _ingress_ traffic. + It can not restrict egress traffic. +* Support for L7 attributes is limited to HTTP method and URL exact/prefix path. +* Supported protocols are limited to TCP-based protocols (for example, HTTP, HTTPS, or gRPC). +* Application layer policies apply to the entire cluster. + They can not be namespaced. +* Logs for application layer polices are not included with other L7 logs in Service Graph. +* Application layer policy is supported only on Kubernetes 1.29 and later. + +:::important + +When you enable application layer policy for a deployment, all of that deployment's pods will restart +Active connections may be disrupted. + +::: + +### Enable application layer policy (ALP) + +Create an (or edit an existing ApplicationLayer) resource with the `sidecarInjection` field to `Enabled`: + +```bash +kubectl apply -f - < -n -p '{"spec":{"template":{"metadata":{"labels":{"applicationlayer.projectcalico.org/sidecar":"true"},"annotations":{"applicationlayer.projectcalico.org/policy":"Enabled"}}}}}' +``` + +### Enforce application layer policies for ingress traffic + +You can restrict ingress traffic using HTTP match criteria using Global network policy. +For a list of all HTTP match parameters, see [Global network policy](/reference/resources/globalnetworkpolicy.mdx). + +In the following example, the trading app is allowed ingress traffic only for HTTP GET requests that match the exact path /projects/calico, or that begins with the prefix, /users. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: customer +spec: + selector: app == 'tradingapp' + ingress: + - action: Allow + http: + methods: ["GET"] + paths: + - exact: "/projects/calico" + - prefix: "/users" + egress: + - action: Allow +``` + +### Disable application layer policies + +To disable the policies, do one of the following steps: +* Recreate your application pods without the sidecar label and annotations. +* Remove the `applicationLayerPolicy` field entirely. +* Remove the `sidecarInjection` field entirely, if sidecar mode was being used only for ALP. +* Delete the ApplicationLayer` custom resource if only ALP was enabled on it. diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/application-layer-policies/index.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/application-layer-policies/index.mdx new file mode 100644 index 0000000000..7318cabe91 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/application-layer-policies/index.mdx @@ -0,0 +1,11 @@ +--- +description: Restrict ingress traffic to Calico Cloud workloads by HTTP method, path, or other Layer-7 attributes using application-layer policy. +hide_table_of_contents: true +--- + +# Application layer policies to control ingress traffic + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/calico-labels.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/calico-labels.mdx new file mode 100644 index 0000000000..6a1c279ddc --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/calico-labels.mdx @@ -0,0 +1,87 @@ +--- +description: Reference list of automatic labels Calico Cloud attaches to resources, useful as selectors in policy rules. +--- + +# Calico Cloud automatic labels + +As a convenience, $[prodname] provides immutable labels that are used for specific resources when evaluating selectors in policies, and works with the labels Kubernetes applies automatically. The labels make it easier to match resources in common ways (such as matching a namespace by name). + +## Labels for matching namespaces + +Kubernetes sets the label `kubernetes.io/metadata.name` to the name of the namespace. This allows for matching namespaces by name when using a `namespaceSelector` field. + +For example, the following GlobalNetworkPolicy applies to workloads with label, `color: red` in namespaces named, `"foo"` and `"bar"`. The policy allows ingress traffic to port 8080 from all workloads in a third namespace named, `"baz"`: + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: foo-and-bar +spec: + namespaceSelector: kubernetes.io/metadata.name in {"foo", "bar"} + selector: color == "red" + types: + - Ingress + ingress: + - action: Allow + source: + namespaceSelector: kubernetes.io/metadata.name == "baz" + destination: + ports: + - 8080 +``` + +Be aware that the default values for `namespaceSelector` for NetworkPolicy and GlobalNetworkPolicy are different. For example: + +**In a network policy**, + + ```yaml + namespaceSelector: + selector: foo == "bar" + ``` +means "resources in the same namespace as the network policy that matches foo == 'bar'". + +**In a global network policy**, + + ```yaml + namespaceSelector: + selector: foo == "bar" + ``` +means "resources in any namespace and non-namespaced resources that match foo == 'bar'". + +Further, + + ```yaml + namespaceSelector: kubernetes.io/metadata.name == "some-namespace" + selector: foo == "bar" + ``` +is equivalent to: + + ```yaml + namespaceSelector: + selector: (foo == "bar") && (projectcalico.org/namespace == "some-namespace") + ``` + +### Labels for matching service accounts + +The `projectcalico.org/name` label is applied to ServiceAccounts and allows for matching by name in a `serviceAccountSelector`. Kubernetes has no equivalent automatic label for ServiceAccounts, so use `projectcalico.org/name` here. + +### The projectcalico.org/name label for namespaces {/* #projectcalico-name-label-for-namespaces */} + +$[prodname] also sets `projectcalico.org/name` to the name of the namespace, and it can be used in a `namespaceSelector` in the same way. It predates the Kubernetes [automatic labeling](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/#automatic-labelling), and $[prodname] derives it as it reads the namespace rather than storing it on the object, so `kubectl get namespace --show-labels` does not show it. For matching namespaces, prefer `kubernetes.io/metadata.name`. + +## Labels for matching host endpoints + +[Automatic HostEndpoints](../../network-policy/hosts/kubernetes-nodes) use the following label to differentiate them from regular HostEndpoints: + +- `projectcalico.org/created-by: calico-kube-controllers` + +## Use the correct selector with labels in policies + +$[prodname] labels must be used with the correct selector or the policy will not work as designed (and there are no error messages in the web console or when applying the YAML). + +| Label | Usage requirements | Use in these resources... | +| --------------------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | +| `kubernetes.io/metadata.name` | Use with a **namespaceSelector**. | - Network policy
- Staged network policy

Namespaced resources that apply only to workload endpoint resources in the namespace.
| +| `projectcalico.org/name` | Use with a **serviceAccountSelector**, or with a **namespaceSelector** (prefer `kubernetes.io/metadata.name` for namespaces). | - Network policy
- Staged network policy

Namespaced resources that apply only to workload endpoint resources in the namespace.
| +| `projectcalico.org/namespace` | Use only with selectors.

Use the label as the label name, and a namespace name as the value to compare against (for example projectcalico.org/namespace == "default"). | - Global network policy
- Staged global network policy

Cluster-wide (non-namespaced) resources that apply to workload endpoint resources in all namespaces, and to host endpoint resources. | diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/calico-network-policy.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/calico-network-policy.mdx new file mode 100644 index 0000000000..8781e0b9ff --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/calico-network-policy.mdx @@ -0,0 +1,256 @@ +--- +description: Write your first Calico Cloud NetworkPolicy — sample policies that exercise the rich rule features beyond Kubernetes NetworkPolicy. +--- + +# Get started with Calico network policy + +## Big picture + +Enforce which network traffic that is allowed or denied using rules in Calico network policy. + +## Value + +### Extends Kubernetes network policy + +Calico network policy provides a richer set of policy capabilities than Kubernetes including: policy ordering/priority, deny rules, and more flexible match rules. While Kubernetes network policy applies only to pods, Calico network policy can be applied to multiple types of endpoints including pods, VMs, and host interfaces. + +### Write once, works everywhere + +No matter which cloud provider you use now, adopting Calico network policy means you write the policy once and it is portable. If you move to a different cloud provider, you don’t need to rewrite your Calico network policy. Calico network policy is a key feature to avoid cloud provider lock-in. + +### Works seamlessly with Kubernetes network policies + +You can use Calico network policy in addition to Kubernetes network policy, or exclusively. For example, you could allow developers to define Kubernetes network policy for their microservices. For broader and higher-level access controls that developers cannot override, you could allow only security or Ops teams to define Calico network policies. + +## Concepts + +### Endpoints + +Calico network policies apply to **endpoints**. In Kubernetes, each pod is a Calico endpoint. However, Calico can support other kinds of endpoints. There are two types of Calico endpoints: **workload endpoints** (such as a Kubernetes pod or OpenStack VM) and **host endpoints** (an interface or group of interfaces on a host). + +### Namespaced and global network policies + +**Calico network policy** is a namespaced resource that applies to pods/containers/VMs in that namespace. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: allow-tcp-port-6379 + namespace: production +``` + +**Calico global network policy** is a non-namespaced resource and can be applied to any kind of endpoint (pods, VMs, host interfaces) independent of namespace. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-tcp-port-6379 +``` + +Because global network policies use **kind: GlobalNetworkPolicy**, they are grouped separately from **kind: NetworkPolicy**. For example, global network policies will not be returned from `kubectl get networkpolicy.p`, and are rather returned from `kubectl get globalnetworkpolicy`. + +### Ingress and egress + +Each network policy rule applies to either **ingress** or **egress** traffic. From the point of view of an endpoint (pod, VM, host interface), **ingress** is incoming traffic to the endpoint, and **egress** is outgoing traffic from the endpoint. In a Calico network policy, you create ingress and egress rules independently (egress, ingress, or both). + +You can specify whether policy applies to ingress, egress, or both using the **types** field. If you do not use the types field, Calico defaults to the following values. + +| Ingress rule present? | Egress rule present? | Value | +| :-------------------: | :------------------: | :-------------: | +| No | No | Ingress | +| Yes | No | Ingress | +| No | Yes | Egress | +| Yes | Yes | Ingress, Egress | + +### Network traffic behaviors: deny and allow + +The Kubernetes network policy specification defines the following behavior: + +- If no network policies apply to a pod, then all traffic to/from that pod is allowed. +- If one or more network policies apply to a pod containing ingress rules, then only the ingress traffic specifically allowed by those policies is allowed. +- If one or more network policies apply to a pod containing egress rules, then only the egress traffic specifically allowed by those policies is allowed. + +For compatibility with Kubernetes, **Calico network policy** follows the same behavior for Kubernetes pods. For other endpoint types (VMs, host interfaces), Calico network policy is default deny. That is, only traffic specifically allowed by network policy is allowed, even if no network policies apply to the endpoint. + +{/*//TODO-CC-XREFS +## Before you begin + +`calicoctl` must be **installed** and **configured** before use. `calicoctl` will use Kubernetes as the datastore. You can find more information on how to configure `calicoctl` in the following link: + +- [Configure `calicoctl`](/operations/clis/calicoctl/configure/overview) +*/} + +## How to + +- [Control traffic to/from endpoints in a namespace](#control-traffic-tofrom-endpoints-in-a-namespace) +- [Control traffic to/from endpoints independent of namespace](#control-traffic-tofrom-endpoints-independent-of-namespace) +- [Control traffic to/from endpoints using IP addresses or CIDR ranges](#control-traffic-tofrom-endpoints-using-ip-addresses-or-cidr-ranges) +- [Apply network policies in specific order](#apply-network-policies-in-specific-order) +- [Generate logs for specific traffic](#generate-logs-for-specific-traffic) + +### Control traffic to/from endpoints in a namespace + +In the following example, ingress traffic to endpoints in the **namespace: production** with label **color: red** is allowed, only if it comes from a pod in the same namespace with **color: blue**, on port **6379**. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: allow-tcp-port-6379 + namespace: production +spec: + selector: color == 'red' + ingress: + - action: Allow + protocol: TCP + source: + selector: color == 'blue' + destination: + ports: + - 6379 +``` + +To allow ingress traffic from endpoints in other namespaces, use a **namespaceSelector** in the policy rule. A namespaceSelector matches namespaces based on the labels that are applied in the namespace. In the following example, ingress traffic is allowed from endpoints in namespaces that match **shape == circle**. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: allow-tcp-port-6379 + namespace: production +spec: + selector: color == 'red' + ingress: + - action: Allow + protocol: TCP + source: + selector: color == 'blue' + namespaceSelector: shape == 'circle' + destination: + ports: + - 6379 +``` + +### Control traffic to/from endpoints independent of namespace + +The following Calico network policy is similar to the previous example, but uses **kind: GlobalNetworkPolicy** so it applies to all endpoints, regardless of namespace. + +In the following example, incoming TCP traffic to any pods with label **color: red** is denied if it comes from a pod with **color: blue**. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: deny-blue +spec: + selector: color == 'red' + ingress: + - action: Deny + protocol: TCP + source: + selector: color == 'blue' +``` + +As with **kind: NetworkPolicy**, you can allow or deny ingress traffic from endpoints in specific namespaces using a namespaceSelector in the policy rule: + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: deny-circle-blue +spec: + selector: color == 'red' + ingress: + - action: Deny + protocol: TCP + source: + selector: color == 'blue' + namespaceSelector: shape == 'circle' +``` + +### Control traffic to/from endpoints using IP addresses or CIDR ranges + +Instead of using a selector to define which traffic is allowed to/from the endpoints in a network policy, you can also specify an IP block in CIDR notation. + +In the following example, outgoing traffic is allowed from pods with the label **color: red** if it goes to an IP address in the **1.2.3.4/24** CIDR block. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: allow-egress-external + namespace: production +spec: + selector: color == 'red' + types: + - Egress + egress: + - action: Allow + destination: + nets: + - 1.2.3.0/24 +``` + +### Apply network policies in specific order + +To control the order/sequence of applying network policies, you can use the **order** field (with precedence from the lowest value to highest). Defining policy **order** is important when you include both **action: allow** and **action: deny** rules that may apply to the same endpoint. + +In the following example, the policy **allow-cluster-internal-ingress** (order: 10) will be applied before the **policy drop-other-ingress** (order: 20). + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: drop-other-ingress +spec: + order: 20 + #...deny policy rules here... +``` + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-cluster-internal-ingress +spec: + order: 10 + #...allow policy rules here... +``` + +### Generate logs for specific traffic + +In the following example, incoming TCP traffic to an application is denied, and each connection attempt is logged to syslog. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: allow-tcp-port-6379 + namespace: production +spec: + selector: role == 'database' + types: + - Ingress + - Egress + ingress: + - action: Log + protocol: TCP + source: + selector: role == 'frontend' + - action: Deny + protocol: TCP + source: + selector: role == 'frontend' +``` + +### Create policy for established connections + +Policies are immediately applied to any new connections. However, for existing connections that are already open, the policy changes will only take effect after the connection has been reestablished. This means that any ongoing sessions may not immediately reflect policy changes until they are initiated again. + +## Additional resources + +- For additional Calico network policy features, see [Calico network policy](../../reference/resources/networkpolicy.mdx) and [Calico global network policy](../../reference/resources/globalnetworkpolicy.mdx) +- For an alternative to using IP addresses or CIDRs in policy, see [Network sets](../../reference/resources/networkset.mdx) +- For details on how to stage network policy, see [Staged network policies](../staged-network-policies.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/index.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/index.mdx new file mode 100644 index 0000000000..5332df5a00 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/index.mdx @@ -0,0 +1,11 @@ +--- +description: Beginner-friendly path for writing your first Calico Cloud network policies — a tour of the basic resource types and rule patterns. +hide_table_of_contents: true +--- + +# Calico Cloud network policy for beginners + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/kubernetes-default-deny.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/kubernetes-default-deny.mdx new file mode 100644 index 0000000000..fd43d0ddfd --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/kubernetes-default-deny.mdx @@ -0,0 +1,149 @@ +--- +description: Apply a default-deny network policy in a Calico Cloud connected cluster so unprotected pods are denied traffic until explicit policy is written. +--- + +# Enable a default deny policy for Kubernetes pods + +## Big picture + +Enable a default deny policy for Kubernetes pods using Kubernetes or $[prodname] network policy. + +## Value + +A **default deny** network policy provides an enhanced security posture so pods without policy (or incorrect policy) are not allowed traffic until appropriate network policy is defined. + +## Features + +This how-to guide uses the following $[prodname] features: + +- **NetworkPolicy** +- **GlobalNetworkPolicy** + +## Concepts + +### Default deny/allow behavior + +**Default allow** means all traffic is allowed by default, unless otherwise specified. **Default deny** means all traffic is denied by default, unless explicitly allowed. **Kubernetes pods are default allow**, unless network policy is defined to specify otherwise. + +For compatibility with Kubernetes, **$[prodname] network policy** enforcement follows the standard convention for Kubernetes pods: + +- If no network policies apply to a pod, then all traffic to/from that pod is allowed. +- If one or more network policies apply to a pod with type ingress, then only the ingress traffic specifically allowed by those policies is allowed. +- If one or more network policies apply to a pod with type egress, then only the egress traffic specifically allowed by those policies is allowed. + +For other endpoint types (VMs, host interfaces), the default behavior is to deny traffic. Only traffic specifically allowed by network policy is allowed, even if no network policies apply to the endpoint. + +## How to + +- [Create a default deny network policy](#create-a-default-deny-network-policy) +- [Create a global default deny network policy](#create-a-global-default-deny-policy) + +### Create a default deny network policy + +Immediately after installation, a best practice is to create a namespaced default deny network policy to secure pods without policy or incorrect policy until you can put policies in place and test them. + +In the following example, we create a $[prodname] default deny **NetworkPolicy** for all workloads in the namespace, **engineering**. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: default-deny + namespace: engineering +spec: + selector: all() + types: + - Ingress + - Egress +``` + +Here's an equivalent default deny **Kubernetes network policy** for all pods in the namespace, **engineering** + +```yaml +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny + namespace: engineering +spec: + podSelector: {} + policyTypes: + - Ingress + - Egress +``` + +### Create a global default deny policy + +A default deny policy ensures that unwanted traffic (ingress and egress) is denied by default without you having to remember default deny/allow behavior of Kubernetes and $[prodname] policies. This policy can also help mitigate risks of lateral malicious attacks. + +#### Best practice #1: Allow, stage, then deny + +We recommend that you create a global default deny policy after you complete writing policy for the traffic that you want to allow. The following steps summarizes the best practice to test and lock down the cluster to block unwanted traffic: + +1. Create a global default deny policy and test it in a staging environment. (The policy will show all the traffic that would be blocked if it were converted into a deny.) +1. Create network policies to individually allow the traffic shown as blocked in step 1 until no connections are denied. +1. Enforce the global default deny policy. + +#### Best practice #2: Keep the scope to non-system pods + +A global default deny policy applies to the entire cluster including all workloads in all namespaces, hosts (computers that run the hypervisor for VMs or container runtime for containers), including Kubernetes control plane and $[prodname] control plane nodes and pods. + +For this reason, the best practice is to create a global default deny policy for **non-system pods** as shown in the following example. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: deny-app-policy +spec: + namespaceSelector: kubernetes.io/metadata.name not in {"calico-system", "kube-public", "kube-system", "tigera-operator"} + types: + - Ingress + - Egress + egress: + # allow all namespaces to communicate to DNS pods + - action: Allow + protocol: UDP + destination: + selector: 'k8s-app == "kube-dns"' + ports: + - 53 + - action: Allow + protocol: TCP + destination: + selector: 'k8s-app == "kube-dns"' + ports: + - 53 +``` + +Note the following: + +- Even though we call this policy "global default deny", the above policy is not explicitly denying traffic. By selecting the traffic with the `namespaceSelector` but not specifying an allow, the traffic is denied after all other policy is evaluated. This design also makes it unnecessary to ensure any specific order (priority) for the default-deny policy. +- Allowing access to `kube-dns` simplifies per-pod policies because you don't need to duplicate the DNS rules in every policy +- This policy uses a negative selector for the `spec.namespaceselector` field to exclude control plane namespaces. + In this example, those namespaces include the `calico-system`, `kube-public`, `kube-system`, and `tigera-operator` namespaces. + Because your installation may have different components, make sure to check what you have installed before implementing a similar policy. + +In a staging environment, verify that the policy does not block any necessary traffic before enforcing it. + +### Don't try this! + +The following policy works and looks fine on the surface. But as described in Best practices #2, the policy is too broad in scope and could break your cluster. Therefore, we do not recommend adding this type of policy, even if you have verified allowed traffic in your staging environment. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: default.default-deny +spec: + tier: default + selector: all() + types: + - Ingress + - Egress +``` + +## Additional resources + +- [Network policy](../../reference/resources/networkpolicy.mdx) +- [Global network policy](../../reference/resources/globalnetworkpolicy.mdx) \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/external-ips-policy.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/external-ips-policy.mdx new file mode 100644 index 0000000000..ea6a34950f --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/external-ips-policy.mdx @@ -0,0 +1,112 @@ +--- +description: Restrict egress and ingress to specific IP ranges in Calico Cloud policy, either inline or via reusable network sets. +--- + +# Use external IPs or networks rules in policy + +## Big picture + +Use $[prodname] network policy to limit traffic to/from external non-$[prodname] workloads or networks. + +## Value + +Modern applications often integrate with third-party APIs and SaaS services that live outside Kubernetes clusters. To securely enable access to those integrations, network security teams must be able to limit IP ranges for egress and ingress traffic to workloads. This includes using IP lists or ranges to deny-list bad actors or embargoed countries. + +Using $[prodname] network policy, you can define IP addresses/CIDRs directly in policy to limit traffic to external networks. Or using $[prodname] network sets, you can easily scale out by using the same set of IPs in multiple policies. + +## Concepts + +### IP addresses/CIDRs + +IP addresses and CIDRs can be specified directly in both Kubernetes and $[prodname] network policy rules. $[prodname] network policy supports IPV4 and IPV6 CIDRs. + +### Network sets + +A **network set** resource is an arbitrary set of IP subnetworks/CIDRs that can be matched by standard label selectors in Kubernetes or $[prodname] network policy. This is useful to reference a set of IP addresses using a selector from a namespaced network policy resource. It is typically used when you want to scale/reuse the same set of IP addresses in policy. + +A **global network set** resource is similar, but can be selected only by $[prodname] global network policies. + +## How to + +- [Limit traffic to or from external networks, IPs in network policy](#limit-traffic-to-or-from-external-networks-ips-in-network-policy) +- [Limit traffic to or from external networks, global network set](#limit-traffic-to-or-from-external-networks-global-network-set) + +### Limit traffic to or from external networks, IPs in network policy + +In the following example, a $[prodname] NetworkPolicy allows egress traffic from pods with the label **color: red**, if it goes to an IP address in the 192.0.2.0/24 CIDR block. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: allow-egress-external + namespace: production +spec: + selector: color == 'red' + types: + - Egress + egress: + - action: Allow + destination: + nets: + - 192.0.2.0/24 +``` + +### Limit traffic to or from external networks, global network set + +In this example, we use a $[prodname] **GlobalNetworkSet** and reference it in a **GlobalNetworkPolicy**. + +In the following example, a $[prodname] **GlobalNetworkSet** deny-lists the CIDR ranges 192.0.2.55/32 and 203.0.113.0/24: + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkSet +metadata: + name: ip-protect + labels: + ip-deny-list: 'true' +spec: + nets: + - 192.0.2.55/32 + - 203.0.113.0/24 +``` + +Next, we create two $[prodname] **GlobalNetworkPolicy** objects. The first is a high “order” policy that allows traffic as a default for things that don’t match our second policy, which is low “order” and uses the **GlobalNetworkSet** label as a selector to deny ingress traffic (IP-deny-list in the previous step). In the label selector, we also include the term **!has(projectcalico.org/namespace)**, which prevents this policy from matching pods or NetworkSets that also have this label. To more quickly enforce the denial of forwarded traffic to the host at the packet level, use the **doNotTrack** and **applyOnForward** options. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: forward-default-allow +spec: + selector: apply-ip-protect == 'true' + order: 1000 + doNotTrack: true + applyOnForward: true + types: + - Ingress + ingress: + - action: Allow +--- +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: ip-protect +spec: + selector: apply-ip-protect == 'true' + order: 0 + doNotTrack: true + applyOnForward: true + types: + - Ingress + ingress: + - action: Deny + source: + selector: ip-deny-list == 'true' && !has(projectcalico.org/namespace) +``` + +## Additional resources + +- To understand how to use global network sets to mitigate common threats, see [Defend against DoS attacks](../../extreme-traffic/defend-dos-attack.mdx) +- [Global network sets](../../../reference/resources/globalnetworkset.mdx) +- [Global network policy](../../../reference/resources/globalnetworkpolicy.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/icmp-ping.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/icmp-ping.mdx new file mode 100644 index 0000000000..40dc912146 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/icmp-ping.mdx @@ -0,0 +1,130 @@ +--- +description: Allow or deny ICMP and ping traffic for Calico Cloud workloads and host endpoints using policy rules. +--- + +# Use ICMP/ping rules in policy + +## Big picture + +Use $[prodname] network policy to allow and deny ICMP/ping messages. + +## Value + +The **Internet Control Message Protocol (ICMP)** provides valuable network diagnostic functions, but it can also be used maliciously. Attackers can use +it to learn about your network, or for DoS attacks. Using $[prodname] network policy, you can control where ICMP is used. For example, you can: + +- Allow ICMP ping, but only for workloads, host endpoints (or both) +- Allow ICMP for pods launched by operators for diagnostic purposes, but block other uses +- Temporarily enable ICMP to diagnose a problem, then disable it after the problem is resolved +- Deny/allow ICMPv4 and/or ICMPv6 + +## Concepts + +### ICMP packet type and code + +$[prodname] network policy also lets you deny and allow ICMP traffic based on specific types and codes. For example, you can specify ICMP type 5, code 2 to match specific ICMP redirect packets. + +For details, see [ICMP type and code](https://en.wikipedia.org/wiki/Internet_Control_Message_Protocol#Control_messages). + +## How to + +- [Deny all ICMP, all workloads and host endpoints](#deny-all-icmp-all-workloads-and-host-endpoints) +- [Allow ICMP ping, all workloads and host endpoints](#allow-icmp-ping-all-workloads-and-host-endpoints) +- [Allow ICMP matching protocol type and code, all Kubernetes pods](#allow-icmp-matching-protocol-type-and-code-all-kubernetes-pods) + +### Deny all ICMP, all workloads and host endpoints + +In this example, we introduce a "deny all ICMP" **GlobalNetworkPolicy**. + +This policy **selects all workloads and host endpoints**. It enables a default deny for all workloads and host endpoints, in addition to the explicit ICMP deny rules specified in the policy. + +If your ultimate goal is to allow some traffic, have your regular "allow" policies in place before applying a global deny-all ICMP traffic policy. + +In this example, all workloads and host endpoints are blocked from sending or receiving **ICMPv4** and **ICMPv6** messages. + +If **ICMPv6** messages are not used in your deployment, it is still good practice to deny them specifically as shown below. + +In any "deny-all" $[prodname] network policy, be sure to specify a lower order (**order:200**) than regular policies that might allow traffic. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: block-icmp +spec: + order: 200 + selector: all() + types: + - Ingress + - Egress + ingress: + - action: Deny + protocol: ICMP + - action: Deny + protocol: ICMPv6 + egress: + - action: Deny + protocol: ICMP + - action: Deny + protocol: ICMPv6 +``` + +### Allow ICMP ping, all workloads and host endpoints + +In this example, workloads and host endpoints can receive **ICMPv4 type 8** and **ICMPv6 type 128** ping requests that come from other workloads and host endpoints. + +All other traffic may be allowed by other policies. If traffic is not explicitly allowed, it will be denied by default. + +The policy applies only to **ingress** traffic. (Egress traffic is not affected, and default deny is not enforced for egress.) + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-ping-in-cluster +spec: + selector: all() + types: + - Ingress + ingress: + - action: Allow + protocol: ICMP + source: + selector: all() + icmp: + type: 8 Ping request + - action: Allow + protocol: ICMPv6 + source: + selector: all() + icmp: + type: 128 Ping request +``` + +### Allow ICMP matching protocol type and code, all Kubernetes pods + +In this example, only Kubernetes pods that match the selector **projectcalico.org/orchestrator == 'kubernetes'** are allowed to receive ICMPv4 **code: 1 host unreachable** messages. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-host-unreachable +spec: + selector: projectcalico.org/orchestrator == 'kubernetes' + types: + - Ingress + ingress: + - action: Allow + protocol: ICMP + icmp: + type: 3 Destination unreachable + code: 1 Host unreachable +``` + +## Additional resources + +For more on the ICMP match criteria, see: + +- [Global network policy](../../../reference/resources/globalnetworkpolicy.mdx) +- [Network policy](../../../reference/resources/networkpolicy.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/index.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/index.mdx new file mode 100644 index 0000000000..38032939ca --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/index.mdx @@ -0,0 +1,11 @@ +--- +description: Control traffic to and from endpoints using Calico Cloud network policy rules — selectors, actions, and egress/ingress directions. +hide_table_of_contents: true +--- + +# Policy rules + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/namespace-policy.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/namespace-policy.mdx new file mode 100644 index 0000000000..2511a1a13b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/namespace-policy.mdx @@ -0,0 +1,89 @@ +--- +description: Group or separate workloads in Calico Cloud policy using namespaces and namespace selectors so policies apply only to specified namespaces. +--- + +# Use namespace rules in policy + +## Big picture + +Use $[prodname] network policies to reference pods in other namespaces. + +## Value + +Kubernetes namespaces let you group/separate resources to meet a variety of use cases. For example, you can use namespaces to separate development, production, and QA environments, or allow different teams to use the same cluster. You can use namespace selectors in $[prodname] network policies to allow or deny traffic to/from pods in specific namespaces. + +## How to + +- [Control traffic to/from endpoints in a namespace](#control-traffic-tofrom-endpoints-in-a-namespace) +- [Use Kubernetes RBAC to control namespace label assignment](#use-kubernetes-rbac-to-control-namespace-label-assignment) + +### Control traffic to/from endpoints in a namespace + +In the following example, ingress traffic is allowed to endpoints in the **namespace: production** with label **color: red**, and only from a pod in the same namespace with **color: blue**, on **port 6379**. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: allow-tcp-6379 + namespace: production +spec: + selector: color == 'red' + ingress: + - action: Allow + protocol: TCP + source: + selector: color == 'blue' + destination: + ports: + - 6379 +``` + +To allow ingress traffic from endpoints in other namespaces, use a **namespaceSelector** in the policy rule. A namespaceSelector matches one or more namespaces based on the labels that are applied on the namespace. In the following example, ingress traffic is also allowed from endpoints with **color: blue** in namespaces with **shape: circle**. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: allow-tcp-6379 + namespace: production +spec: + selector: color == 'red' + ingress: + - action: Allow + protocol: TCP + source: + selector: color == 'blue' + namespaceSelector: shape == 'circle' + destination: + ports: + - 6379 +``` + +### Use Kubernetes RBAC to control namespace label assignment + +Network policies can be applied to endpoints using selectors that match labels on the endpoint, the endpoint's namespace, or the endpoint's service account. By applying selectors based on the endpoint's namespace, you can use Kubernetes RBAC to control which users can assign labels to namespaces. This allows you to separate groups who can deploy pods from those who can assign labels to namespaces. + +In the following example, users in the development environment can communicate only with pods that have a namespace labeled, `environment == "development"`. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: restrict-development-access +spec: + namespaceSelector: 'environment == "development"' + ingress: + - action: Allow + source: + namespaceSelector: 'environment == "development"' + egress: + - action: Allow + destination: + namespaceSelector: 'environment == "development"' +``` + +## Additional resources + +- For more network policies, see [Network policy](../../../reference/resources/networkpolicy.mdx) +- To apply policy to all namespaces, see [Global network policy](../../../reference/resources/globalnetworkpolicy.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/policy-rules-overview.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/policy-rules-overview.mdx new file mode 100644 index 0000000000..16988f5f1e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/policy-rules-overview.mdx @@ -0,0 +1,22 @@ +--- +description: How to write policy rules in Calico Cloud — label selectors, source and destination match criteria, and rule actions. +--- + +# Basic rules + +## Big picture + +Use Calico policy rules and label selectors that match Calico endpoints (pods, OpenStack VMs, and host interfaces) to define network connectivity. + +## Value + +Using label selectors to identify the endpoints (pods, OpenStack VMs, host interfaces) that a policy applies to, or that should be selected by policy rules, means you can define policy without knowing the IP addresses of the endpoints. This is ideal for handling dynamic workloads with ephemeral IPs (such as Kubernetes pods). + +## How to + +Read [Get started with Calico policy](../calico-network-policy.mdx) and [Kubernetes policy](../../../tutorials/kubernetes-tutorials/kubernetes-network-policy.mdx), which cover all the basics of using label selectors in policies to select endpoints the policies apply to, or in policy rules. + +## Additional resources + +- [Global network policy](../../../reference/resources/globalnetworkpolicy.mdx) +- [Network policy](../../../reference/resources/networkpolicy.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/service-accounts.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/service-accounts.mdx new file mode 100644 index 0000000000..5fe275ef6b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/service-accounts.mdx @@ -0,0 +1,116 @@ +--- +description: Match on Kubernetes service accounts in Calico Cloud policy rules to validate workload identity and apply RBAC-controlled rules. +--- + +# Use service accounts rules in policy + +## Big picture + +Use $[prodname] network policy to allow/deny traffic for Kubernetes service accounts. + +## Value + +Using $[prodname] network policy, you can leverage Kubernetes service accounts with RBAC for flexible control over how policies are applied in a cluster. For example, the security team can have RBAC permissions to: + +- Control which service accounts the developer team can use within a namespace +- Write high-priority network policies for those service accounts (that the developer team cannot override) + +The network security team can maintain full control of security, while selectively allowing developer operations where it makes sense. + +## Concepts + +### Use smallest set of permissions required + +Operations on service accounts are controlled by RBAC, so you can grant permissions only to trusted entities (code and/or people) to create, modify, or delete service accounts. To perform any operation in a workload, clients are required to authenticate with the Kubernetes API server. + +If you do not explicitly assign a service account to a pod, it uses the default ServiceAccount in the namespace. + +You should not grant broad permissions to the default service account for a namespace. If an application needs access to the Kubernetes API, create separate service accounts with the smallest set of permissions required. + +### Service account labels + +Like all other Kubernetes objects, service accounts have labels. You can use labels to create ‘groups’ of service accounts. $[prodname] network policy lets you select workloads by their service account using: + +- An exact match on service account name +- A service account label selector expression + +## Before you begin... + +Configure unique Kubernetes service accounts for your applications. + +## How to + +- [Limit ingress traffic for workloads by service account name](#limit-ingress-traffic-for-workloads-by-service-account-name) +- [Limit ingress traffic for workloads by service account label](#limit-ingress-traffic-for-workloads-by-service-account-label) +- [Use Kubernetes RBAC to control service account label assignment](#use-kubernetes-rbac-to-control-service-account-label-assignment) + +### Limit ingress traffic for workloads by service account name + +In the following example, ingress traffic is allowed from any workload whose service account matches the names **api-service** or **user-auth-service**. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: demo-calico + namespace: prod-engineering +spec: + ingress: + - action: Allow + source: + serviceAccounts: + names: + - api-service + - user-auth-service + selector: 'app == "db"' +``` + +### Limit ingress traffic for workloads by service account label + +In the following example, ingress traffic is allowed from any workload whose service account matches the label selector, **app == web-frontend**. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: allow-web-frontend + namespace: prod-engineering +spec: + ingress: + - action: Allow + source: + serviceAccounts: + selector: 'app == "web-frontend"' + selector: 'app == "db"' +``` + +### Use Kubernetes RBAC to control service account label assignment + +Network policies can be applied to endpoints using selectors that match labels on the endpoint, the endpoint's namespace, or the endpoint's service account. By applying selectors based on the endpoint's service account, you can use Kubernetes RBAC to control which users can assign labels to service accounts. This allows you to separate groups who can deploy pods from those who can assign labels to service accounts. + +In the following example, pods with an intern service account can communicate only with pods with service accounts labeled, `role: intern`. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: restrict-intern-access + namespace: prod-engineering +spec: + serviceAccountSelector: 'role == "intern"' + ingress: + - action: Allow + source: + serviceAccounts: + selector: 'role == "intern"' + egress: + - action: Allow + destination: + serviceAccounts: + selector: 'role == "intern"' +``` + +## Additional resources + +- [Network policy](../../../reference/resources/networkpolicy.mdx) +- [Global network policy](../../../reference/resources/globalnetworkpolicy.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/service-policy.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/service-policy.mdx new file mode 100644 index 0000000000..f778334f71 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/policy-rules/service-policy.mdx @@ -0,0 +1,119 @@ +--- +description: Match on Kubernetes Service names in Calico Cloud policy rules instead of specific pod selectors. +--- + +# Use service rules in policy + +## Big picture + +Use $[prodname] network policy to allow/deny traffic for Kubernetes services. + +## Value + +Using $[prodname] network policy, you can leverage Kubernetes Service names to easily define access to Kubernetes services. Using service names in policy enables you to: + +- Allow or deny access to the Kubernetes API service. +- Reference port information already declared by the application, making it easier to keep policy up-to-date as application requirements change. + +## How to + +- [Allow access to the Kubernetes API for a specific namespace](#allow-access-to-the-kubernetes-api-for-a-specific-namespace) +- [Allow access to Kubernetes DNS for the entire cluster](#allow-access-to-kubernetes-dns-for-the-entire-cluster) +- [Allow access from a specified service](#allow-access-from-a-specified-service) + +### Allow access to the Kubernetes API for a specific namespace + +In the following example, egress traffic is allowed to the `kubernetes` service in the `default` namespace for all pods in the namespace `my-app`. This service is the typical +access point for the Kubernetes API server. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: allow-api-access + namespace: my-app +spec: + selector: all() + egress: + - action: Allow + destination: + services: + name: kubernetes + namespace: default +``` + +Endpoint addresses and ports to allow will be automatically detected from the service. + +### Allow access to Kubernetes DNS for the entire cluster + +In the following example, a GlobalNetworkPolicy is used to select all pods in the cluster to apply a rule which ensures +all pods can access the Kubernetes DNS service. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-kube-dns +spec: + selector: all() + egress: + - action: Allow + destination: + services: + name: kube-dns + namespace: kube-system +``` + +:::note + +This policy also enacts a default-deny behavior for all pods, so make sure any other required application traffic is allowed by a policy. + +::: + +## Allow access from a specified service + +In the following example, ingress traffic is allowed from the `frontend-service` service in the `frontend` namespace for all pods in the namespace `backend`. +This allows all pods that back the `frontend-service` service to send traffic to all pods in the `backend` namespace. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: allow-frontend-service-access + namespace: backend +spec: + selector: all() + ingress: + - action: Allow + source: + services: + name: frontend-service + namespace: frontend +``` + +We can also further specify the ports that the `frontend-service` service is allowed to access. The following example limits access from the `frontend-service` +service to port 80. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: allow-frontend-service-access + namespace: backend +spec: + selector: all() + ingress: + - action: Allow + protocol: TCP + source: + services: + name: frontend-service + namespace: frontend + destination: + ports: [80] +``` + +## Additional resources + +- [Network policy](../../../reference/resources/networkpolicy.mdx) +- [Global network policy](../../../reference/resources/globalnetworkpolicy.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/services/index.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/services/index.mdx new file mode 100644 index 0000000000..65c9a0223d --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/services/index.mdx @@ -0,0 +1,11 @@ +--- +description: Apply Calico Cloud policy to Kubernetes Services — node ports, ClusterIPs, and externally exposed services. +hide_table_of_contents: true +--- + +# Policy for Kubernetes services + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/services/kubernetes-node-ports.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/services/kubernetes-node-ports.mdx new file mode 100644 index 0000000000..74a2979189 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/services/kubernetes-node-ports.mdx @@ -0,0 +1,135 @@ +--- +description: Restrict access to Kubernetes NodePort services using a Calico Cloud GlobalNetworkPolicy at the host endpoint. +--- + +# Apply Calico Cloud policy to Kubernetes node ports + +## Big picture + +Restrict access to node ports to specific external clients. + +## Value + +Exposing services to external clients using node ports is a standard Kubernetes feature. However, if you want to restrict access to node ports to specific external clients, you need to use Calico global network policy. + +## Concepts + +### Network policy with preDNAT field + +In a Kubernetes cluster, kube-proxy will DNAT a request to the node's port and IP address to one of the pods that backs the service. For Calico global network policy to both allow normal ingress cluster traffic and deny other general ingress traffic, it must take effect before DNAT. To do this, you simply add a **preDNAT** field to a Calico global network policy. The preDNAT field: + +- Applies before DNAT +- Applies only to ingress rules +- Enforces all ingress traffic through a host endpoint, regardless of destination + The destination can be a locally hosted pod, a pod on another node, or a process running on the host. + +## Before you begin... + +For services that you want to expose to external clients, configure Kubernetes services with type **NodePort**. + +## How to + +To securely expose a Kubernetes service to external clients, you must implement all of the following steps. + +- [Allow cluster ingress traffic, but deny general ingress traffic](#allow-cluster-ingress-traffic-but-deny-general-ingress-traffic) +- [Allow local host egress traffic](#allow-local-host-egress-traffic) +- [Create host endpoints with appropriate network policy](#create-host-endpoints-with-appropriate-network-policy) +- [Allow ingress traffic to specific node ports](#allow-ingress-traffic-to-specific-node-ports) + +### Allow cluster ingress traffic but deny general ingress traffic + +In the following example, we create a global network policy to allow cluster ingress traffic (**allow-cluster-internal-ingress**): for the nodes’ IP addresses (**1.2.3.4/16**), and for pod IP addresses assigned by Kubernetes (**100.100.100.0/16**). By adding a preDNAT field, Calico global network policy is applied before regular DNAT on the Kubernetes cluster. + +In this example, we use the **selector: has(kubernetes-host)** -- so the policy is applicable to any endpoint with a **kubernetes-host** label (but you can easily specify particular nodes). + +Finally, when you specify a preDNAT field, you must also add the **applyOnForward: true** field. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-cluster-internal-ingress-only +spec: + order: 20 + preDNAT: true + applyOnForward: true + ingress: + - action: Allow + source: + nets: [1.2.3.4/16, 100.100.100.0/16] + - action: Deny + selector: has(kubernetes-host) +``` + +### Allow local host egress traffic + +We also need a global network policy to allow egress traffic through each node's external interface. Otherwise, when we define host endpoints for those interfaces, no egress traffic will be allowed from local processes (except for traffic that is allowed by the [Failsafe rules](../../../reference/host-endpoints/failsafe.mdx). + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-outbound-external +spec: + order: 10 + egress: + - action: Allow + selector: has(kubernetes-host) +``` + +### Create host endpoints with appropriate network policy + +In this example, we assume that you have already defined Calico host endpoints with network policy that is appropriate for the cluster. (For example, you wouldn’t want a host endpoint with a “default deny all traffic to/from this host” network policy because that is counter to the goal of allowing/denying specific traffic.) For help, see [host endpoints](../../../reference/resources/hostendpoint.mdx). + +All of our previously-defined global network policies have a selector that makes them applicable to any endpoint with a **kubernetes-host label**; so we will include that label in our definitions. For example, for **eth0** on **node1**. + +```yaml +apiVersion: projectcalico.org/v3 +kind: HostEndpoint +metadata: + name: node1-eth0 + labels: + kubernetes-host: ingress +spec: + interfaceName: eth0 + node: node1 + expectedIPs: + - INSERT_IP_HERE +``` + +When creating each host endpoint, replace `INSERT_IP_HERE` with the IP address on eth0. The `expectedIPs` field is required so that any selectors within ingress or egress rules can properly match the host endpoint. + +### Allow ingress traffic to specific node ports + +Now we can allow external access to the node ports by creating a global network policy with the preDNAT field. In this example, **ingress traffic is allowed** for any host endpoint with **port: 31852**. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-nodeport +spec: + preDNAT: true + applyOnForward: true + order: 10 + ingress: + - action: Allow + protocol: TCP + destination: + selector: has(kubernetes-host) + ports: [31852] + selector: has(kubernetes-host) +``` + +To make the NodePort accessible only through particular nodes, give the nodes a particular label. For example: + +```yaml +nodeport-external-ingress: true +``` + +Then, use **nodeport-external-ingress: true** as the selector of the **allow-nodeport** policy, instead of **has(kubernetes-host)**. + +## Additional resources + +- [Global network policy](../../../reference/resources/globalnetworkpolicy.mdx) +- [Host endpoints](../../../reference/resources/hostendpoint.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/services/services-cluster-ips.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/services/services-cluster-ips.mdx new file mode 100644 index 0000000000..09d376055e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/services/services-cluster-ips.mdx @@ -0,0 +1,193 @@ +--- +description: Expose Kubernetes Service ClusterIPs over BGP using Calico Cloud and restrict who can reach them with network policy. +--- + +# Apply Calico Cloud policy to services exposed externally as cluster IPs + +## Big picture + +Control access to services exposed through clusterIPs that are advertised outside the cluster using BGP. + +## Value + +$[prodname] network policy uses standard Kubernetes Services that allow you to expose services within clusters to external clients in the following ways: + +- [Apply policy to Kubernetes nodeports](kubernetes-node-ports.mdx) +- Using cluster IPs over BGP (described in this article) + +## Concepts + +### Advertise cluster IPs outside the cluster + +A **cluster IP** is a virtual IP address that represents a Kubernetes Service. Kube Proxy on each host translates the clusterIP into a pod IP for one of the pods backing the service, acting as a reverse proxy and load balancer. + +Cluster IPs were originally designed for use within the Kubernetes cluster. $[prodname] allows you to advertise Cluster IPs externally -- so external clients can use them to access services hosted inside the cluster. This means that $[prodname] ingress policy can be applied at **one or both** of the following locations: + +- Host interface, when the traffic destined for the clusterIP first ingresses the cluster +- Pod interface of the backend pod + +### Traffic routing: local versus cluster modes + +$[prodname] implements [Kubernetes service external traffic policy](https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip), which controls whether external traffic is routed to node-local or cluster-wide endpoints. The following table summarizes key differences between these settings. The default is **cluster mode**. + +| **Service setting** | **Traffic is load balanced...** | **Pros and cons** | **Required service type** | +| ------------------------------------------- | --------------------------------------------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | +| **externalTrafficPolicy: Cluster**(default) | Across all nodes in the cluster | Equal distribution of traffic among all pods running a service.

Possible unnecessary network hops between nodes for ingress external traffic.When packets are rerouted to pods on another node, traffic is SNAT’d (source network address translation).

Destination pod can see the proxying node’s IP address rather than the actual client IP. | **ClusterIP** | +| **externalTrafficPolicy: Local** | Across the nodes with the endpoints for the service | Avoids extra hops so better for apps that ingress a lot external traffic.

Traffic is not SNAT’d so actual client IPs are preserved.

Traffic distributed among pods running a service may be imbalanced. | **LoadBalancer** (for cloud providers), or **NodePort** (for node’s static port) | + +## Before you begin... + +[Configure Calico to advertise cluster IPs over BGP](../../../networking/configuring/advertise-service-ips.mdx). + +## How to + +Selecting which mode to use depends on your goals and resources. At an operational level, **local mode** simplifies policy, but load balancing may be uneven in certain scenarios. **Cluster mode** requires more work to manage clusterIPs, SNAT, and create policies that reference specific IP addresses, but you always get even load balancing. + +- [Secure externally exposed cluster IPs, local mode](#secure-externally-exposed-cluster-ips-local-mode) +- [Secure externally exposed cluster IPs, cluster mode](#secure-externally-exposed-cluster-ips-cluster-mode) + +### Secure externally exposed cluster IPs, local mode + +Using **local mode**, the original source address of external traffic is preserved, and you can define policy directly using standard $[prodname] network policy. + +1. Create $[prodname] **NetworkPolicies** or **GlobalNetworkPolicies** that select the same set of pods as your Kubernetes Service. +1. Add rules to allow the external traffic. +1. If desired, add rules to allow in-cluster traffic. + +### Secure externally exposed cluster IPs, cluster mode + +In the following steps, we define **GlobalNetworkPolicy** and **HostEndpoints**. + +#### Step 1: Verify Kubernetes Service manifest + +Ensure that your Kubernetes Service manifest explicitly lists the clusterIP; do not allow Kubernetes to automatically assign the clusterIP because you need it for your policies in the following steps. + +#### Step 2: Create global network policy at the host interface + +In this step, you create a **GlobalNetworkPolicy** that selects all **host endpoints**. It controls access to the cluster IP, and prevents unauthorized clients from outside the cluster from accessing it. The hosts then forwards only authorized traffic. + +**Set policy to allow external traffic for cluster IPs** + +Add rules to allow the external traffic for each clusterIP. The following example allows connections to two cluster IPs. Make sure you add **applyOnForward** and **preDNAT** rules. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-cluster-ips +spec: + selector: k8s-role == 'node' + types: + - Ingress + applyOnForward: true + preDNAT: true + ingress: + # Allow 50.60.0.0/16 to access Cluster IP A + - action: Allow + source: + nets: + - 50.60.0.0/16 + destination: + nets: + - 10.20.30.40/32 Cluster IP A + # Allow 70.80.90.0/24 to access Cluster IP B + - action: Allow + source: + nets: + - 70.80.90.0/24 + destination: + nets: + - 10.20.30.41/32 Cluster IP B +``` + +**Add a rule to allow traffic destined for the pod CIDR** + +Without this rule, normal pod-to-pod traffic is blocked because the policy applies to forwarded traffic. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-to-pods +spec: + selector: k8s-role == 'node' + types: + - Ingress + applyOnForward: true + preDNAT: true + ingress: + # Allow traffic forwarded to pods + - action: Allow + destination: + nets: + - 192.168.0.0/16 Pod CIDR +``` + +**Add a rule to allow traffic destined for all host endpoints** + +Or, you can add rules that allow specific host traffic including Kubernetes and $[prodname]. Without this rule, normal host traffic is blocked. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-traffic-hostendpoints +spec: + selector: k8s-role == 'node' + types: + - Ingress + # Allow traffic to the node (not nodePorts, TCP) (not nodePorts, TCP) + - action: Allow + protocol: TCP + destination: + selector: k8s-role == 'node' + notPorts: ["30000:32767"] #nodePort range + # Allow traffic to the node (not nodePorts, TCP) (not nodePorts, UDP) + - action: Allow + protocol: UDP + destination: + selector: k8s-role == 'node' + notPorts: ["30000:32767"] #nodePort range +``` + +#### Step 3: Create a global network policy that selects pods + +In this step, you create a **GlobalNetworkPolicy** that selects the **same set of pods as your Kubernetes Service**. Add rules that allow host endpoints to access the service ports. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-nodes-svc-a +spec: + selector: k8s-svc == 'svc-a' + types: + - Ingress + ingress: + - action: Allow + protocol: TCP + source: + selector: k8s-role == 'node' + destination: + ports: [80, 443] + - action: Allow + protocol: UDP + source: + selector: k8s-role == 'node' + destination: + ports: [80, 443] +``` + +#### Step 4: (Optional) Create network polices or global network policies that allow in-cluster traffic to access the service + +#### Step 5: Create HostEndpoints + +Create HostEndpoints for the interface of each host that will receive traffic for the clusterIPs. Be sure to label them so they are selected by the policy in Step 2 (Add a rule to allow traffic destined for the pod CIDR), and the rules in Step 3. + +In the previous example policies, the label **k8s-role: node** is used to identify these HostEndpoints. + +## Additional resources + +- [Enable service IP advertisement](../../../networking/configuring/advertise-service-ips.mdx) +- [Defend against DoS attacks](../../extreme-traffic/defend-dos-attack.mdx) +- [Global network policy](../../../reference/resources/globalnetworkpolicy.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/simple-policy-cnx.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/simple-policy-cnx.mdx new file mode 100644 index 0000000000..fdb7654ba2 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/beginners/simple-policy-cnx.mdx @@ -0,0 +1,331 @@ +--- +description: Tour of the additional features Calico Cloud adds to Kubernetes policy that make it suitable for production environments. +--- + +# Calico Cloud for Kubernetes demo + +This guide is a variation of the [simple policy demo](../../tutorials/kubernetes-tutorials/kubernetes-policy-basic.mdx) intended to introduce the extra features of $[prodname] to people already familiar with Project Calico for Kubernetes. + +It requires a Kubernetes cluster configured with Calico networking and $[prodname], and expects that you have `kubectl` configured to interact with the cluster. + +{/*//TODO-CC-XREFS +You can quickly and easily obtain such a cluster by following one of the +[installation guides](/getting-started/install-on-clusters/kubernetes/index), +or by [upgrading an existing cluster](/getting-started/upgrading/upgrading-calico-to-calico-enterprise/upgrade-to-tsee/index). + +The key steps in moving to $[prodname] are to change to the $[prodname] version of calico-node, update its configuration, download [calicoq](/reference/clis/calicoq/index) and deploy Prometheus. +*/} + +This guide assumes that you have installed all the $[prodname] components from the +guides above and that your cluster consists of the following nodes: + +- k8s-node1 +- k8s-node2 +- k8s-master + +Where you see references to these in the text below, substitute for your actual node names. You can find what nodes are on your cluster with `kubectl get nodes` + +## Configure Namespaces + +This guide will deploy pods in a Kubernetes namespace. Let's create the `Namespace` object for this guide. + +``` +kubectl create ns policy-demo +``` + +## Create demo pods + +We'll use Kubernetes `Deployment` objects to easily create pods in the namespace. + +1. Create some nginx pods in the `policy-demo` namespace. + + ```shell + kubectl create deployment --namespace=policy-demo nginx --image=nginx + ``` + +1. Expose them through a service. + + ```shell + kubectl expose --namespace=policy-demo deployment nginx --port=80 + ``` + +1. Ensure the nginx service is accessible. + + ```shell + kubectl run --namespace=policy-demo access --rm -ti --image busybox /bin/sh + ``` + + This should open up a shell session inside the `access` pod, as shown below. + + ``` + Waiting for pod policy-demo/access-472357175-y0m47 to be running, status is Pending, pod ready: false + + If you don't see a command prompt, try pressing enter. + + / # + ``` + +1. From inside the `access` pod, attempt to reach the `nginx` service. + + ```shell + wget -q nginx -O - + ``` + + You should see a response from `nginx`. Great! Our service is accessible. You can exit the pod now. + +1. Inspect the network policies using calicoq. The `host` command displays + information about the policies for endpoints on a given host. + + :::note + + calicoq complements calicoctl by inspecting the + dynamic aspects of $[prodname] Policy: in particular displaying the endpoints actually affected by policies, + and the policies that actually apply to endpoints. + {/*//TODO-CC-XREFS The full calicoq documentation is [here](/reference/clis/calicoq/index). */} + + ::: + + ``` + DATASTORE_TYPE=kubernetes calicoq host k8s-node1 + ``` + + You should see the following output. + + ``` + Policies and profiles for each endpoint on host "k8s-node1": + + Workload endpoint k8s/tigera-prometheus.alertmanager-calico-node-alertmanager-0/eth0 + Policies: + Policy "tigera-prometheus/knp.default.calico-node-alertmanager" (order 1000; selector "(projectcalico.org/orchestrator == 'k8s' && alertmanager == 'calico-node-alertmanager' && app == 'alertmanager') && projectcalico.org/namespace == 'tigera-prometheus'") + Policy "tigera-prometheus/knp.default.calico-node-alertmanager-mesh" (order 1000; selector "(projectcalico.org/orchestrator == 'k8s' && alertmanager == 'calico-node-alertmanager' && app == 'alertmanager') && projectcalico.org/namespace == 'tigera-prometheus'") + Policy "tigera-prometheus/knp.default.default-deny" (order 1000; selector "(projectcalico.org/orchestrator == 'k8s') && projectcalico.org/namespace == 'tigera-prometheus'") + Profiles: + Profile "kns.tigera-prometheus" + Rule matches: + Policy "tigera-prometheus/knp.default.calico-node-alertmanager-mesh" inbound rule 1 source match; selector "(projectcalico.org/namespace == 'tigera-prometheus') && (projectcalico.org/orchestrator == 'k8s' && app in { 'alertmanager' } && alertmanager in { 'calico-node-alertmanager' })" + + ... + + Workload endpoint k8s/policy-demo.nginx-8586cf59-5bxvh/eth0 + Policies: + Profiles: + Profile "kns.policy-demo" + ``` + + For each workload endpoint, the `Policies:` section lists the policies that + apply to that endpoint, in the order they apply. calicoq displays both + $[prodname] Policies and Kubernetes NetworkPolicies, although this + example focuses on the latter. The `Rule matches:` section lists the + policies that match that endpoint in their rules, in other words that have + rules that deny or allow that endpoint as a packet source or destination. + + Focusing on the + `k8s/tigera-prometheus.alertmanager-calico-node-alertmanager-0/eth0` endpoint: + + - The first two policies are defined in the monitor-calico.yaml manifest. + The selectors here have been translated from the original NetworkPolicies to + the $[prodname] format (note the addition of the namespace test). + + - The third policy and the following profile are created automatically by the + policy controller. + +1. Use kubectl to see the detail of any particular policy or profile. For + example, for the `kns.policy-demo` profile, which defines default behavior for + pods in the `policy-demo` namespace: + + ```shell + kubectl get profile kns.policy-demo -o yaml + ``` + + You should see the following output. + + ```yaml + apiVersion: projectcalico.org/v3 + kind: Profile + metadata: + creationTimestamp: '2022-01-06T21:32:05Z' + name: kns.policy-demo + resourceVersion: 435026/ + uid: 75dd2ed4-d3a6-41ca-a106-db073bfa946a + spec: + egress: + - action: Allow + destination: {} + source: {} + ingress: + - action: Allow + destination: {} + source: {} + labelsToApply: + pcns.projectcalico.org/name: policy-demo + ``` + + Alternatively, you may also use the $[prodname] web console to inspect and view information and metrics associated with policies, endpoints, and nodes. + +## Enable isolation + +Let's turn on isolation in our policy-demo namespace. $[prodname] will then prevent connections to pods in this namespace. + +Running the following command creates a NetworkPolicy which implements a default deny behavior for all pods in the `policy-demo` namespace. + +```shell +kubectl create -f - < + +### Workload and host endpoints + +Policy with domain names can be enforced on workload or host endpoints. When a policy with domain names applies to a workload endpoint, it +allows that workload to connect out to the specified domains. When policy with domain names applies to a host endpoint, it allows clients +directly on the relevant host (including any host-networked workloads) to connect out to the specified domains. + +### Trusted DNS servers + +$[prodname] trusts DNS information only from its list of DNS trusted servers. Using trusted DNS servers to back domain names in +policy, prevents a malicious workload from using IPs returned by a fake DNS server to hijack domain names in policy rules. + +By default, $[prodname] trusts the Kubernetes cluster’s DNS service (kube-dns or CoreDNS). For workload endpoints, these +out-of-the-box defaults work with standard Kubernetes installs, so normally you won’t change them. For host endpoints you will need to add +the IP addresses that the cluster nodes use for DNS resolution. + +## Before you begin + +**Not supported** + +DNS policy is not supported at egress of egress gateway pods. Domain-based rules will either never match in +that hook, or, they may match intermittently. Intermittent matches occur when a pod on the same node as the +egress gateway pod happens to make a matching DNS query. This is because the DNS-to-IP cache used to render +the policy is shared node-wide. + +## How to + +You can specify allowed domain names directly in a **global network policy** or **namespaced network policy**, or specify domain names in a **global network set** (and then +reference the global network set in a global network policy). + +- [Use domain names in a global network policy](#use-domain-names-in-a-global-network-policy) +- [Use domain names in a namespaced network policy](#use-domain-names-in-a-namespaced-network-policy) +- [Use domain names in a global network set, reference the set in a global network policy](#use-domain-names-in-a-global-network-set) + +### Best practice + +Use a **global network set** when the same set of domains needs to be referenced in multiple policies, or when you want the allowed +destinations to be a mix of domains and IPs from global network sets, or IPs from workload endpoints and host endpoints. By using a single +destination selector in a global network set, you can potentially match all of these resources. + +### Use domain names in a global network policy + +In this method, you create a **GlobalNetworkPolicy** with egress rules with `action: Allow` and a `destination.domains` field specifying the +domain names to which egress traffic is allowed. + +In the following example, the first rule allows DNS traffic, and the second rule allows connections outside the cluster to domains +**api.alice.com** and **\*.example.com** (which means `.example.com`, such as **bob.example.com**). + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-egress-to-domains +spec: + order: 1 + selector: my-pod-label == 'my-value' + types: + - Egress + egress: + - action: Allow + protocol: UDP + destination: + ports: + - 53 + - dns + - action: Allow + destination: + domains: + - api.alice.com + - '*.example.com' +``` + +### Use domain names in a namespaced network policy + +In this method, you create a **NetworkPolicy** with egress rules with `action: Allow` and a `destination.domains` field specifying the +domain names to which egress traffic is allowed. + +In the following example, the first rule allows DNS traffic, and the second rule allows connections outside the cluster to domains +**api.alice.com** and **\*.example.com** (which means `.example.com`, such as **bob.example.com**). + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: allow-egress-to-domains + namespace: rollout-test +spec: + order: 1 + selector: my-pod-label == 'my-value' + types: + - Egress + egress: + - action: Allow + protocol: UDP + destination: + ports: + - 53 + - dns + - action: Allow + destination: + domains: + - api.alice.com + - '*.example.com' +``` + +The difference between this and the **GlobalNetworkPolicy** example is that this namespaced NetworkPolicy can only grant egress access, to the specified domains, to workload endpoints in the `rollout-test` namespace. + +### Use domain names in a global network set + +In this method, you create a **GlobalNetworkSet** with the allowed destination domain names in the `allowedEgressDomains` field. Then, +you create a **GlobalNetworkPolicy** with a `destination.selector` that matches that GlobalNetworkSet. + +In the following example, the allowed egress domains (`api.alice.com` and `*.example.com`) are specified in the GlobalNetworkSet. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkSet +metadata: + name: allowed-domains-1 + labels: + color: red +spec: + allowedEgressDomains: + - api.alice.com + - '*.example.com' +``` + +Then, you reference the global network set in a **GlobalNetworkPolicy** using a destination label selector. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-egress-to-domain +spec: + order: 1 + selector: my-pod-label == 'my-value' + types: + - Egress + egress: + - action: Allow + destination: + selector: color == 'red' +``` + +### Use domain names in a network set + +In this method, you create a **NetworkSet** with the allowed destination domain names in the `allowedEgressDomains` field. Then, +you create a **NetworkPolicy** with a `destination.selector` that matches that NetworkSet. + +In the following example, the allowed egress domains (`api.alice.com` and `*.example.com`) are specified in the NetworkSet. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkSet +metadata: + name: allowed-domains-1 + namespace: rollout-test + labels: + color: red +spec: + allowedEgressDomains: + - api.alice.com + - '*.example.com' +``` + +Then, you reference the network set in a **NetworkPolicy** using a destination label selector. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: allow-egress-to-domain + namespace: rollout-test +spec: + order: 1 + selector: my-pod-label == 'my-value' + types: + - Egress + egress: + - action: Allow + destination: + selector: color == 'red' +``` + +### Tuning performance and latency + +$[prodname] supports different DNS policy modes with different performance and latency implications. +The default mode for the iptables and nftables data planes is `DelayDeniedPacket`. +The eBPF data plane defaults to `Inline` on kernels 5.17 (or RedHat 5.14) and newer and to `NoDelay` on older kernels. +The Windows data plane support only `NoDelay` mode. + +With `NoDelay` mode, new pods sometimes fail to connect to domains that are allowed by a DNS policy. +In these cases, the pod tries to connect to a domain before $[prodname] has finished processing the DNS policy that allows the connection. +As soon as the processing is complete, the pod is able to connect. + +For more information, see [DNSPolicyMode](../reference/resources/felixconfig#dnspolicymode) and [BPFDNSPolicyMode](../reference/resources/felixconfig#bpfdnspolicymode). + +## Additional resources + +To change the default DNS trusted servers, use the [DNSTrustedServers parameter](../reference/component-resources/node/felix/configuration.mdx). + +For more detail about the relevant resources, see +[GlobalNetworkSet](../reference/resources/globalnetworkset.mdx), +[GlobalNetworkPolicy](../reference/resources/globalnetworkpolicy.mdx), +[NetworkPolicy](../reference/resources/networkpolicy.mdx) +and +[NetworkSet](../reference/resources/networkset.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/extreme-traffic/defend-dos-attack.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/extreme-traffic/defend-dos-attack.mdx new file mode 100644 index 0000000000..a89587e152 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/extreme-traffic/defend-dos-attack.mdx @@ -0,0 +1,107 @@ +--- +description: Define DoS mitigation rules in Calico Cloud policy that drop connections at the eBPF or XDP layer, with hardware offload when available. +--- + +# Defend against DoS attacks + +## Big picture + +Calico automatically enforces specific types of deny-list policies at the earliest possible point in the packet processing pipeline, including offloading to NIC hardware whenever possible. + +## Value + +During a DoS attack, a cluster can receive massive numbers of connection requests from attackers. The faster these connection requests are dropped, the less flooding and overloading to your hosts. When you define DoS mitigation rules in Calico network policy, Calico enforces the rules as efficiently as possible to minimize the impact. + +## Concepts + +### Earliest packet processing + +The earliest point in the packet processing pipeline that packets can be dropped, depends on the Linux kernel version and the capabilities of the NIC driver and NIC hardware. Calico automatically uses the fastest available option. + +| Processed by... | Used by Calico if... | Performance | +| --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | +| NIC hardware | The NIC supports **XDP offload** mode. | Fastest | +| NIC driver | The NIC driver supports **XDP native** mode. | Faster | +| Kernel | The kernel supports **XDP generic mode** and Calico is configured to explicitly use it. This mode is rarely used and has no performance benefits over iptables raw mode below. To enable, see [Felix Configuration](../../reference/resources/felixconfig.mdx). | Fast | +| Kernel | If none of the modes above are available, **iptables raw** mode is used. | Fast | + +:::note + +XDP modes require Linux kernel v4.16 or later. + +::: + +## How to + +The high-level steps to defend against a DoS attack are: + +- [Step 1: Create host endpoints](#step-1-create-host-endpoints) +- [Step 2: Add CIDRs to deny-list in a global network set](#step-2-add-cidrs-to-deny-list-in-a-global-network-set) +- [Step 3: Create deny incoming traffic global network policy](#step-3-create-deny-incoming-traffic-global-network-policy) + +### Best practice + +The following procedure covers the required steps, assuming no prior configuration is in place. A best practice is to proactively do these steps before an attack (create the host endpoints, network policy, and global network set). In the event of a DoS attack, you can quickly respond by just adding the CIDRs that you want to deny-list to the global network set. + +### Step 1: Create host endpoints + +First, you create the HostEndpoints corresponding to the network interfaces where you want to enforce DoS mitigation rules. In the following example, the HostEndpoint secures the interface named **eth0** with IP **10.0.0.1** on node **jasper**. + +```yaml +apiVersion: projectcalico.org/v3 +kind: HostEndpoint +metadata: + name: production-host + labels: + apply-dos-mitigation: 'true' +spec: + interfaceName: eth0 + node: jasper + expectedIPs: ['10.0.0.1'] +``` + +### Step 2: Add CIDRs to deny-list in a global network set + +Next, you create a Calico **GlobalNetworkset**, adding the CIDRs that you want to deny-list. In the following example, the global network set deny-lists the CIDR ranges **1.2.3.4/32** and **5.6.0.0/16**: + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkSet +metadata: + name: dos-mitigation + labels: + dos-deny-list: 'true' +spec: + nets: + - '1.2.3.4/32' + - '5.6.0.0/16' +``` + +### Step 3: Create deny incoming traffic global network policy + +Finally, create a Calico GlobalNetworkPolicy adding the GlobalNetworkSet label (**dos-deny-list** in the previous step) as a selector to deny ingress traffic. To more quickly enforce the denial of forwarded traffic to the host at the packet level, use the **doNotTrack** and **applyOnForward** options. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: dos-mitigation +spec: + selector: apply-dos-mitigation == 'true' + doNotTrack: true + applyOnForward: true + types: + - Ingress + ingress: + - action: Deny + source: + selector: dos-deny-list == 'true' +``` + +## Additional resources + +- [Global network sets](../../reference/resources/globalnetworkset.mdx) +- [Global network policy](../../reference/resources/globalnetworkpolicy.mdx) +- [Create a host endpoint](../../reference/resources/hostendpoint.mdx) +- [Introduction to XDP](https://www.iovisor.org/technology/xdp) +- [Advanced XDP documentation](https://prototype-kernel.readthedocs.io/en/latest/networking/XDP/index.html) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/extreme-traffic/high-connection-workloads.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/extreme-traffic/high-connection-workloads.mdx new file mode 100644 index 0000000000..941db9113b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/extreme-traffic/high-connection-workloads.mdx @@ -0,0 +1,89 @@ +--- +description: Bypass Linux conntrack with a Calico Cloud policy rule for workloads that handle an extreme number of concurrent connections. +--- + +# Enable extreme high-connection workloads + +## Big picture + +Use a $[prodname] network policy rule to bypass Linux conntrack for traffic to workloads that experience extremely large number of connections. + +## Value + +When the number of connections on a node exceeds the number of connections that Linux conntrack can track, connections can be rejected or dropped. $[prodname] network policy can be used to selectively bypass Linux conntrack for traffic to/from these types of workloads. + +## Concepts + +### Linux conntrack + +Connection tracking (“conntrack”) is a core feature of the Linux kernel’s networking stack. It allows the kernel to keep track of all logical network connections or flows, and thereby identify all of the packets that make up each flow so they can be handled consistently together. Conntrack is an essential part of the mainline Linux network processing pipeline, normally improving performance, and enabling NAT and stateful access control. + +### Extreme high-connection workloads + +Some niche workloads handling extremely high number of simultaneous connections, or very high rate of short lived connections, can exceed the maximum number of connections Linux conntrack is able to track. One real world example of such a workload is an extreme scale memcached server handling 50k+ connections per second. + +### $[prodname] doNotTrack network policy + +The $[prodname] global network policy option, **doNotTrack**, indicates to apply the rules in the policy before connection tracking, and that packets allowed by these rules should not be tracked. The policy is applied early in the Linux packet processing pipeline, before any regular network policy rules, and independent of the policy order field. + +Unlike normal network policy rules, doNotTrack network policy rules are stateless, meaning you must explicitly specify rules to allow return traffic that would normally be automatically allowed by conntrack. For example, for a server on port 999, the policy must include an ingress rule allowing inbound traffic to port 999, and an egress rule to allow outbound traffic from port 999. + +In a doNotTrack policy: + +- Ingress rules apply to all incoming traffic through a host endpoint, regardless of where the traffic is going +- Egress rules apply only to traffic that is sent from the host endpoint (not a local workload) + +Finally, you must add an **applyOnForward: true expression** for a **doNotTrack policy** to work. + +## Before you begin... + +Before creating a **doNotTrack** network policy, read this [blog](https://www.tigera.io/blog/when-linux-conntrack-is-no-longer-your-friend/) to understand use cases, benefits, and trade offs. + +## How to + +### Bypass connection traffic for high connection server + +In the following example, a memcached server pod with **hostNetwork: true** was scheduled on the node memcached-node-1. We create a HostEndpoint for the node. Next, we create a GlobalNetwork Policy with symmetrical rules for ingress and egress with doNotTrack and applyOnForward set to true. + +```yaml +apiVersion: projectcalico.org/v3 +kind: HostEndpoint +metadata: + name: memcached-node-1-eth0 + labels: + memcached: server +spec: + interfaceName: eth0 + node: memcached-node-1 + expectedIPs: + - 10.128.0.162 +--- +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: memcached-server +spec: + selector: memcached == 'server' + applyOnForward: true + doNotTrack: true + ingress: + - action: Allow + protocol: TCP + source: + selector: memcached == 'client' + destination: + ports: + - 12211 + egress: + - action: Allow + protocol: TCP + source: + ports: + - 12211 + destination: + selector: memcached == 'client' +``` + +## Additional resources + +[Global network policy](../../reference/resources/globalnetworkpolicy.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/extreme-traffic/index.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/extreme-traffic/index.mdx new file mode 100644 index 0000000000..03ea89212f --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/extreme-traffic/index.mdx @@ -0,0 +1,11 @@ +--- +description: Apply Calico Cloud network policy early in the Linux packet-processing pipeline to handle DoS, high-connection, and other extreme traffic scenarios. +hide_table_of_contents: true +--- + +# Policy for extreme traffic + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/hosts/host-forwarded-traffic.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/hosts/host-forwarded-traffic.mdx new file mode 100644 index 0000000000..98ee8b428a --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/hosts/host-forwarded-traffic.mdx @@ -0,0 +1,292 @@ +--- +description: Apply Calico Cloud network policy to traffic forwarded through hosts acting as routers or NAT gateways. +--- + +# Apply policy to forwarded traffic + +## Big picture + +Enforce $[prodname] policy on traffic transiting a host that is used as a router or NAT gateway. + +## Value + +If your host has multiple network interfaces, and is configured as a router, or NAT gateway between two different networks, you may want to enforce policy on traffic as it moves between the networks. In this configuration, often neither the source or destination is a $[prodname] endpoint, so policy enforcement at the endpoint is not available. You can centrally manage the firewall policy on a fleet of such hosts using the same policy language as the rest of $[prodname]. + +## Concepts + +### Workload endpoints and host endpoints + +The following figure shows a host with two network interfaces: eth0 and eth1. We call these **host endpoints (HEPs)**. The host also runs two guest workloads (VMs or containers). We call the virtual interfaces to the guests, **workload endpoints (WEPs)**. Each has a corresponding configuration object on the $[prodname] API called HostEndpoint and WorkloadEndpoint, respectively. + +The `HostEndpoint API` object is optional, and $[prodname] does not enforce any policy on the HEP if the API object is missing. The `WorkloadEndpoint API` object is required, and is automatically managed by the cluster orchestrator plugin (for example, Kubernetes or OpenStack). + +Several connections are shown in the figure, numbered 1 through 4. For example, connection 1 ingresses over HEP eth0, is forwarded, and then ingresses Workload A’s WEP. Calico policies select which WEPs or HEPs they apply to. So, for example an ingress policy that selects Workload A’s WEP will apply to connections as shown in number 1. + +![Host-forward-traffic](/img/calico-enterprise/host-forward-traffic.png) + +### applyOnForward + +By default, $[prodname] global network policies set **applyOnForward to false**. When set to false on policies that select HEPs, the policies are applied only to traffic that originates or terminates on the host, for example: connection 4 (Node process). Connections 1-3 are unaffected by policies that select the HEP, but have applyOnForward set to false. + +In contrast, if applyOnForward is set to true for a policy that selects a HEP, that policy can apply to all connections 1-4. For example: + +- Ingress policy on HEP eth0 affects connections 1 and 2 +- Egress policy on HEP eth1 affects connections 2, 3, and 4 + +There are also different default action semantics for **applyOnForward: true policy** versus **applyOnForward: false policy**. +An applyOnForward: true policy affects all traffic through the HEP (connections 1-4). If no applyOnForward policy selects the HEP and direction (ingress versus egress), then forwarded traffic is allowed. If no policy (regardless of applyOnForward) selects the HEP and direction, then local traffic is denied. + +| **HEP defined?** | **Traffic Type** | **applyOnForward defined?** | **Any policy defined?** | **Default Action** | +| ---------------- | ---------------- | --------------------------- | ----------------------- | ------------------ | +| No | Any | n/a | n/a | Allow | +| Yes | Forwarded | No | Any | Allow | +| Yes | Forwarded | Yes | Yes | Deny | +| Yes | Local | n/a | No | Deny | +| Yes | Local | n/a | Yes | Deny | + +**$[prodname] namespaced network policies** do not have an applyOnForward setting. HEPs are always cluster global, not namespaced, so network policies cannot select them. + +### preDNAT policy + +Hosts are often configured to perform Destination Network Address Translation before forwarding certain packets. A common example of this in cloud computing is when the host acts as a reverse-proxy to load balance service requests for a set of backend workload instances. To apply policy to a specific example of such a reverse-proxy, see [Kubernetes nodePorts](../beginners/services/kubernetes-node-ports.mdx). + +When preDNAT is set to false on a global network policy, the policy rules are evaluated on the connection after DNAT is performed. False is the default. When preDNAT is set to true, the policy rules are evaluated on the connection before DNAT has been performed. + +If you set preDNAT policy to true, you must set applyOnForward to true, and preDNAT policy must only include Ingress policies. + +### Host endpoints with interfaceName: `*` + +HostEndpoint API objects can be created with the name of the host interface (as reported by ip link or similar), or they can be created with interfaceName set to `*`, which means all host interfaces on the given node, including the interfaces between the host to any WEPs on that host. + +With interfaceName set to a particular interface, any policies that select the HEP apply only if the traffic goes through the named interface. With it set to `*`, policies that select the HEP apply regardless of the interface. + +This is particularly relevant when you want to enforce policy for a host that also runs guest workloads like VMs or Pods. Traffic from local workloads to reverse-proxy IPs or ports do not traverse any external interfaces, and thus a HEP with interfaceName set to \* is required in order for policy to apply to them. + +## How to + +### Control forwarded traffic in or out of particular networks + +1. Choose a labeling scheme for your Host Endpoints (network interfaces). + For example, if you have an application network and management network, you might choose the labels **network = application** and **network = management**. +1. Write GlobalNetworkPolicies expressing your desired rules. + - applyOnForward set to true. + - Use the **selector:** to choose which Host Endpoints to apply policy. +1. Create the HostEndpoint objects on the `$[prodname] API`. + - Label the HostEndpoints according to the label scheme you developed in step 1. + - We recommend that you create policies before you create the Host Endpoints. This ensures that all policies exist before $[prodname] starts enforcing. + +## Tutorial + +Let’s say I have a host that has two network interfaces: + +- eth0 - connects to the main datacenter network for application traffic +- eth1 - connects to a special maintenance network + +My goal is to allow SSH traffic to be forwarded to the maintenance network, but to drop all other traffic. + +I choose the following label scheme: + +- network = application for the main datacenter network for application traffic +- network = maintenance for the maintenance network + +I create the GlobalNetworkPolicy that allows SSH traffic (default deny is implicit in this case). + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-ssh-maintenance +spec: + selector: network == 'maintenance' + applyOnForward: true + types: + - Ingress + ingress: + # Allow SSH + - action: Allow + protocol: TCP + destination: + ports: + - 22 +``` + +Save this as allow-ssh-maintenance.yaml. + +Apply the policy to the cluster: + +```bash +kubectl create -f allow-ssh-maintenance.yaml +``` + +Finally, create the host endpoint for the interface that connects to the maintenance network. + +```yaml +apiVersion: projectcalico.org/v3 +kind: HostEndpoint +metadata: + name: myhost.eth1 + labels: + network: maintenance +spec: + interfaceName: eth1 + node: myhost + expectedIPs: + - 192.168.0.45 +``` + +Replace myhost with the node name $[prodname] uses, and the expected IPs with the actual interface IP address(es). Save this file as hep.yaml. + +Apply the host endpoint to the cluster: + +```bash +kubectl create -f hep.yaml +``` + +For completeness, you could also create a HostEndpoint for eth0, but because we have not written any policies for the application network yet, you can omit this step. + +## Viewing flow logs for host forwarded traffic + +When using `applyOnForward` and `preDNAT` policies with host endpoints, you can monitor and troubleshoot policy enforcement by examining flow logs. $[prodname] captures detailed information about how policies are applied to forwarded traffic. + +### Configure policy information in flow logs + +To see policy information in flow logs, you must enable flow logs for forwarding decisions in the FelixConfiguration resource: + +```yaml +apiVersion: projectcalico.org/v3 +kind: FelixConfiguration +metadata: + name: default +spec: + flowLogsPolicyScope: AllPolicies +``` + +This configuration enables tracking the flows transiting through the node interfaces within `policies` field in flow logs. The `policies` field contains detailed information about which policies acted on the traffic. + +### Understanding the `transit_policies` field in flow logs + +The `policies` field in flow logs contains sub-fields that show different policy evaluation results: + +- **`transit_policies`**: All host endpoint policies with `applyOnForward: true` that applied an action to the flow while in transit through the node (for forwarded traffic between network interfaces). Note that policies with `preDNAT: true` will also appear here when they apply to ingress traffic before DNAT processing. + +Each policy entry in these fields follows this format: +``` +||.|| +``` + +### Example flow log for applyOnForward policy + +Using the tutorial example from this document (SSH traffic to maintenance network), here's what a flow log entry might look like: + +```json +{ + "source_ip": "10.0.0.1", + "source_name": "client-pods-gkh45", + "source_name_aggr": "client-pods-*", + "source_namespace": "client-pods-namespace", + "dest_ip": "192.168.1.10", + "dest_name": "maintenance-server-gkh45", + "dest_name_aggr": "maintenance-server-*", + "dest_namespace": "maintenance-server-namespace", + "dest_port": 22, + "dest_type": "wep", + "proto": "tcp", + "action": "allow", + "reporter": "dst, fwd", + "policies": { + "all_policies": [ + "0|__PROFILE__|__PROFILE__.kns.maintenance-server|allow|0" + ], + "enforced_policies": [ + "0|__PROFILE__|__PROFILE__.kns.maintenance-server|allow|0" + ], + "pending_policies": [ + "0|__PROFILE__|__PROFILE__.kns.maintenance-server|allow|0" + ], + "transit_policies": [ + "0|default|default.allow-ssh-maintenance|allow|0" + ] + }, + "bytes_in": 1234, + "bytes_out": 5678, + "packets_in": 10, + "packets_out": 8, + "host": "my-host" +} +``` + +**Key points about this flow log:** +- `reporter` is `dst, fwd`, indicating this flow was reported by both the destination workload endpoint and the forwarding node. The `enforced_policies` shows policies that applied to the destination workload's ingress traffic, while `transit_policies` shows the host endpoint policy that applied as the traffic was forwarded through the node. +- The `policies.transit_policies` field shows that the `allow-ssh-maintenance` GlobalNetworkPolicy allowed the traffic +- The rule index `0` indicates the first rule in the policy matched (the SSH allow rule) + +### Example flow log for preDNAT policy + +Here's an example of a GlobalNetworkPolicy with `preDNAT: true` that allows HTTP traffic incoming to the cluster from external sources: + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-external-http +spec: + selector: network = 'maintenance' + applyOnForward: true + preDNAT: true + types: + - Ingress + ingress: + # Allow HTTP from external sources to NodePort services + - action: Allow + protocol: TCP + source: + nets: + - 34.0.113.0/24 # External network + destination: + ports: + - 30080 # NodePort +``` + +For preDNAT policies, flow logs display the original destination IP and port before DNAT is performed. When traffic originates from outside the cluster (for example, an external client accesses a NodePort service), the flow log captures the pre-DNAT source and destination details. Here's an example: + +```json +{ + "source_ip": "34.0.113.100", + "source_name": "-", + "source_name_aggr": "pub", + "source_namespace": "-", + "source_type": "net", + "dest_ip": "66.11.155.1", + "dest_name": "-", + "dest_name_aggr": "pub", + "dest_namespace": "-", + "dest_type": "net", + "dest_port": 30080, + "proto": "tcp", + "action": "allow", + "reporter": "fwd", + "policies": { + "transit_policies": [ + "0|default|default.allow-external-http|allow|0" + ] + }, + "bytes_in": 2048, + "bytes_out": 4096, + "packets_in": 15, + "packets_out": 12, + "host": "my-host" +} +``` + +**Key points about this preDNAT flow log:** +- `source_name_aggr` is `pub`, indicating external traffic from outside the cluster +- `dest_name_aggr` is `pub`, indicating the NodePort's external facing IP +- `dest_port` shows the original NodePort (30080) before DNAT translation to the service port +- The `reporter` is `fwd`, indicating this policy acted on forwarded ingress traffic at the host endpoint +- The `policies.transit_policies` field shows the preDNAT policy that allowed the traffic before DNAT processing + +## Additional resources + +- [Host endpoint](../../reference/resources/hostendpoint.mdx) +- [Workload endpoint](../../reference/resources/workloadendpoint.mdx) +- [Global network policy](../../reference/resources/globalnetworkpolicy.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/hosts/kubernetes-nodes.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/hosts/kubernetes-nodes.mdx new file mode 100644 index 0000000000..5e7f813b3b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/hosts/kubernetes-nodes.mdx @@ -0,0 +1,296 @@ +--- +description: Protect Kubernetes node interfaces with Calico Cloud host endpoints to extend network policy to the node itself. +--- + +# Protect Kubernetes nodes + +## Big picture + +Secure Kubernetes nodes with host endpoints managed by $[prodname]. + +## Value + +$[prodname] can automatically create host endpoints for your Kubernetes nodes. This means $[prodname] can manage the lifecycle of host endpoints as your cluster evolves, ensuring nodes are always protected by policy. + +## Concepts + +## Host endpoints + +Each host has one or more network interfaces that it uses to communicate externally. You can represent these interfaces in $[prodname] using host endpoints and then use network policy to secure them. + +$[prodname] host endpoints can have labels, and they work the same as labels on workload endpoints. The network policy rules can apply to both workload and host endpoints using label selectors. + +Automatic host endpoints secure all of the host's interfaces (i.e. in Linux, all the interfaces in the host network namespace). They are created by setting `interfaceName: "*"`. + +## Automatic host endpoints + +$[prodname] creates a wildcard host endpoint for each node, with the host endpoint containing the same labels and IP addresses as its corresponding node. +$[prodname] will ensure these managed host endpoints maintain the same labels and IP addresses as its node by periodic syncs. +This means that policy targeting these automatic host endpoints will function correctly with the policy put in place to select those nodes, even if over time the node's IPs or labels change. + +$[prodname] is also able to create and manage custom host endpoints for nodes, based on user specified templates. +This allows you to fine-tune which interfaces should be included in the host endpoint and for which nodes these host endpoints should be created. +The host endpoint templates can be enabled by updating [KubeControllersConfig](../../reference/resources/kubecontrollersconfig.mdx). + +Automatic host endpoints are differentiated from other host endpoints by the label `projectcalico.org/created-by: calico-kube-controllers`. +Enable or disable automatic host endpoints by configuring the default KubeControllersConfiguration resource. + +## Before you begin + +**Unsupported** + +- GKE + +## How to + +- [Enable automatic host endpoints](#enable-automatic-host-endpoints) +- [Apply network policy to automatic host endpoints](#apply-network-policy-to-automatic-host-endpoints) + +### Enable automatic host endpoints + +To enable automatic host endpoints, edit the default KubeControllersConfiguration instance, and set `spec.controllers.node.hostEndpoint.autoCreate` to `true`: + +```bash +kubectl patch kubecontrollersconfiguration default --patch='{"spec": {"controllers": {"node": {"hostEndpoint": {"autoCreate": "Enabled"}}}}}' +``` + +If successful, host endpoints are created for each of your cluster's nodes: + +```bash +kubectl get heps -o wide +``` + +The output may look similar to this: + +``` +kubectl get heps -o wide +NAME CREATED AT +ip-172-16-101-147.us-west-2.compute.internal-auto-hep 2021-05-12T22:16:47Z +ip-172-16-101-54.us-west-2.compute.internal-auto-hep 2021-05-12T22:16:47Z +ip-172-16-101-79.us-west-2.compute.internal-auto-hep 2021-05-12T22:16:47Z +ip-172-16-101-9.us-west-2.compute.internal-auto-hep 2021-05-12T22:16:47Z +ip-172-16-102-63.us-west-2.compute.internal-auto-hep 2021-05-12T22:16:47Z +``` + +### Use custom templates for host endpoints + +Template HostEndpoints allow for tailored, automatic HostEndpoint generation. Defined in spec.controllers.node.hostEndpoint.templates, they enable creation based on specific Node or interface criteria, offering greater customization compared to default automatic HostEndpoints. + +To enable template host endpoints and disable default host endpoints, edit the default KubeControllersConfiguration instance, and update `spec.controllers.node.hostEndpoint.templates` with your custom templates spec. + +1. Create a custom templates spec: + ```bash + cat << 'EOF' > hostEndpointTemplate.yaml + spec: + controllers: + node: + hostEndpoint: + autoCreate: Enabled + createDefaultHostEndpoint: Disabled + templates: + - generateName: custom-host-endpoint + interfaceCIDRs: + - 192.168.0.0/16 + nodeSelector: "!has(node-role.kubernetes.io/control-plane)" + labels: + custom-label: template-host-endpoint + annotations: + custom-annotation: template-host-endpoint + EOF + ``` + +2. Merge the templates spec with the `kubecontrollersconfiguration` resource: + + ```bash + kubectl patch kubecontrollersconfiguration default --patch-file hostEndpointTemplate.yaml --type merge + ``` + +3. Verify that your `kubecontrollersconfiguration` resource is updated: + + ```bash + kubectl get kubecontrollersconfiguration default -o yaml + ``` + + The output should look similar to this: + + ``` + apiVersion: projectcalico.org/v3 + kind: KubeControllersConfiguration + metadata: + name: default + spec: + controllers: + node: + hostEndpoint: + autoCreate: Enabled + createDefaultHostEndpoint: Enabled + templates: + - generateName: custom-host-endpoint + interfaceCIDRs: + - 192.168.0.0/16 + labels: + custom-label: template-host-endpoint + annotations: + custom-annotation: template-host-endpoint + ``` + +4. If successful, host endpoints are created for nodes matching your node selector: + + ```bash + kubectl get heps -o=custom-columns='NAME:.metadata.name,NODE:.spec.node,INTERFACES:.spec.interfaceName,IPS:.spec.expectedIPs,PROFILES:.spec.profiles' + ``` + + The output may look similar to this: + + ``` + NAME NODE INTERFACE IPS PROFILES + 192.168.83.64.compute.internal-custom-host-endpoint-auto-hep 192.168.83.64.compute.internal 192.168.83.64 projectcalico-default-allow + 192.168.0.128.compute.internal-custom-host-endpoint-auto-hep 192.168.0.128.compute.internal 192.168.0.128 projectcalico-default-allow + 192.168.157.128.compute.internal-custom-host-endpoint-auto-hep 192.168.157.128.compute.internal 192.168.157.128 projectcalico-default-allow + ``` + +### Apply network policy to automatic host endpoints + +To apply policy that targets all Kubernetes nodes, first add a label to the nodes. +The label will be synced to their automatic host endpoints. + +For example, to add the label **kubernetes-host** to all nodes and their host endpoints: + +```bash +kubectl label nodes --all kubernetes-host= +``` + +And an example policy snippet: + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: all-nodes-policy +spec: + selector: has(kubernetes-host) + # +``` + +To select a specific set of host endpoints (and their corresponding Kubernetes nodes), use a policy selector that selects a label unique to that set of host endpoints. +For example, if we want to add the label **environment=dev** to nodes named node1 and node2: + +```bash +kubectl label node node1 environment=dev +kubectl label node node2 environment=dev +``` + +With the labels in place and automatic host endpoints enabled, host endpoints for node1 and node2 will be updated with the **environment=dev** label. +We can write policy to select that set of nodes with a combination of selectors: + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: some-nodes-policy +spec: + selector: has(kubernetes-host) && environment == 'dev' + # +``` + +## Tutorial + +This tutorial will lock down Kubernetes node ingress to only allow SSH and required ports for Kubernetes to function. +We will apply two policies: one for the control plane nodes. and one for the worker nodes. + +:::note + +Note: This tutorial was tested on a cluster created with kubeadm v1.18.2 on AWS, using a "stacked etcd" [topology](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/ha-topology/). Stacked etcd topology means the etcd pods are running on the masters. kubeadm uses stacked etcd by default. +If your Kubernetes cluster is on a different platform, is running a variant of Kubernetes, or is running a topology with an external etcd cluster, +please review the required ports for control plane and worker nodes in your cluster and adjust the policies in this tutorial as needed. + +::: + +First, let's restrict ingress traffic to the control plane nodes. The ingress policy below contains three rules. +The first rule allows access to the API server port from anywhere. The second rule allows all traffic to localhost, which +allows Kubernetes to access control plane processes. These control plane processes includes the etcd server client API, the scheduler, and the controller-manager. +This rule also allows localhost access to the kubelet API and calico/node health checks. +And the final rule allows the etcd pods to peer with each other and allows the masters to access each others kubelet API. + +If you have not modified the failsafe ports, you should still have SSH access to the nodes after applying this policy. +Now apply the ingress policy for the Kubernetes masters: + +``` +kubectl apply -f - << EOF +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: ingress-k8s-masters +spec: + selector: has(node-role.kubernetes.io/master) + # This rule allows ingress to the Kubernetes API server. + ingress: + - action: Allow + protocol: TCP + destination: + ports: + # kube API server + - 6443 + # This rule allows all traffic to localhost. + - action: Allow + destination: + nets: + - 127.0.0.0/8 + # This rule is required in multi-master clusters where etcd pods are colocated with the masters. + # Allow the etcd pods on the masters to communicate with each other. 2380 is the etcd peer port. + # This rule also allows the masters to access the kubelet API on other masters (including itself). + - action: Allow + protocol: TCP + source: + selector: has(node-role.kubernetes.io/master) + destination: + ports: + - 2380 + - 10250 +EOF +``` + +Note that the above policy selects the standard **node-role.kubernetes.io/master** label that kubeadm sets on control plane nodes. + +Next, we need to apply policy to restrict ingress to the Kubernetes workers. +Before adding the policy we will add a label to all of our worker nodes, which then gets added to its automatic host endpoint. +For this tutorial we will use **kubernetes-worker**. An example command to add the label to worker nodes: + +```bash +kubectl get node -l '!node-role.kubernetes.io/master' -o custom-columns=NAME:.metadata.name | tail -n +2 | xargs -I{} kubectl label node {} kubernetes-worker= +``` + +The workers' ingress policy consists of two rules. The first rule allows all traffic to localhost. As with the masters, +the worker nodes need to access their localhost kubelet API and calico/node healthcheck. +The second rule allows the masters to access the workers kubelet API. Now apply the policy: + +``` +kubectl apply -f - << EOF +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: ingress-k8s-workers +spec: + selector: has(kubernetes-worker) + # Allow all traffic to localhost. + ingress: + - action: Allow + destination: + nets: + - 127.0.0.0/8 + # Allow only the masters access to the nodes kubelet API. + - action: Allow + protocol: TCP + source: + selector: has(node-role.kubernetes.io/master) + destination: + ports: + - 10250 +EOF +``` + +## Additional resources + +- [Apply policy to Kubernetes node ports](../beginners/services/kubernetes-node-ports.mdx) +- [Global network policy](../../reference/resources/globalnetworkpolicy.mdx) +- [Host endpoints](../../reference/resources/hostendpoint.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/index.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/index.mdx new file mode 100644 index 0000000000..624ec3680f --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/index.mdx @@ -0,0 +1,55 @@ +--- +description: Secure Kubernetes workloads and hosts in connected clusters with Calico Cloud network policy — managed enforcement, tiers, recommendations, and observability. +--- + +import { DocCardLink, DocCardLinkLayout } from '/src/___new___/components'; + +# Network policy + +Writing network policies is how you restrict traffic to pods in your Kubernetes cluster. +$[prodname] extends the standard `NetworkPolicy` object to provide advanced network policy features, such as policies that apply to all namespaces. + +## Getting started + + + + + + + + + + +## Policy rules + + + + + + + + + + +## Policy tiers + + + + + + + + +## Policy for services + + + + + + +## Policy for extreme traffic + + + + + \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/networksets.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/networksets.mdx new file mode 100644 index 0000000000..d6378a9dbd --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/networksets.mdx @@ -0,0 +1,267 @@ +--- +description: Use Calico Cloud network sets to package frequently reused IP ranges or domains into named selectors that policies can reference across connected clusters. +--- + +# Get started with network sets + +## Visualize traffic to/from your cluster + +Modern applications often integrate with third-party APIs and SaaS services that live outside Kubernetes clusters. To securely enable access to those integrations, you must be able to limit IP ranges for egress and ingress traffic to workloads. Limiting IP lists or ranges is also used to deny-list bad actors or embargoed countries. To limit IP ranges, you need to use the $[prodname] resource called **network sets**. + +## What are network sets? + +**Network sets** are a grouping mechanism that allows you to create an arbitrary set of IP subnetworks/CIDRs or domains that can be matched by standard label selectors in Kubernetes or $[prodname] network policy. Like IP pools for pods, they allow you to reuse/scale sets of IP addresses in policies. + +A **network set** is a namespaced resource that you can use with Kubernetes or $[prodname] network policies; a **global network set** is a cluster-wide resource that you can use with $[prodname] network policies. + +Like network policy, you manage user access to network sets using standard Kubernetes RBAC. + +## Why are network sets powerful? + +If you are familiar with Service Graph in the web console, you know the value of seeing pod-to-pod traffic within your cluster. But what about traffic external to your cluster? + +$[prodname] automatically detects IPs for pods and nodes that fall into the standard IETF “public network” and “private network” designations, and displays those as icons in Service Graph. So you get some visibility into external traffic without using any network sets. + +![public-private-networks](/img/calico-enterprise/public-private-networks.png) + +However, when you create network sets, you can get more granular visibility into what's leaving the cluster to public networks. Because you control the grouping, the naming, and labeling, you create visibility that is customized to your organization. This is why they are so powerful. + +Here are just a few examples of how network sets can be used: + +- **Egress access control** + + Network sets are a key resource for defining egress access controls; for example, securing ingress to microservices/apps or egress from workloads outside the cluster. + +- **Troubleshooting** + + Network sets appear as additional metadata in flow logs, Flow Visualizer, and Service Graph. + +- **Efficiency and scaling** + + Network sets are critical when scaling your deployment. You may have only a few CIDRs when you start. But as you scale out, it is easier to update a handful of network sets than update each network policy individually. Also, in a Kubernetes deployment, putting lots of anything (CIDRs, ports, policy rules) directly into policies causes inefficiencies in traffic processing (iptables/eBPF). + +- **Microsegmentation and shift left** + + Network sets provide the same microsegmentation controls as network policy. For example, you can allow specific users to create policies (that reference network sets), but allow only certain users to manage network sets. + +- **Threat defense** + + Network sets are key to being able to manage threats by blocking bad IPs with policy in a timely way. Imagine having to update individual policies when you find a bad IP you need to quickly block. You can even give access to a controller that automatically updates CIDRs in a network set when a bad IP is found. + +## Create a network set and use it in policy + +In this section, we’ll walk through how to create a namespaced network set in the web console. You can follow along using your cluster or tigera-labs cluster. + +In this example, you will create a network set named, `google`. This network set contains a list of trusted google endpoints for a microservice called, `hipstershop`. As a service owner, you want to be able to see traffic leaving the microservices in Service Graph. Instead of matching endpoints on IP addresses, we will use domain names. + +1. From the left navbar, click **Network Sets**. +1. Click **Add Network Set**, and enter these values. + - For Name: `google` + - For Scope: Select **Namespace** and select, `hipstershop` +1. Under Labels, click **Add label**. + - In the Select key field, enter `destinations` and click the green bar to add this new entry. + - In the Value field, enter `google`, click the green bar to add the entry, and save. +1. For Domains, click **+Add Domain** and these URLs: `clouddebugger.googleapis.com`, `cloudtrace.googleapis.com`, `metadata.google.internal`, `monitoring.googleapis.com`. +1. Click **Create Network Set**. + +You’ve created your first network set. + +![add-networkset-google](/img/calico-enterprise/add-networkset-google.png) + +The YAML looks like this: + +```yaml +kind: NetworkSet +apiVersion: projectcalico.org/v3 +metadata: + name: google + labels: + destinations: google + namespace: hipstershop +spec: + nets: [] + allowedEgressDomains: + - clouddebugger.googleapis.com + - cloudtrace.googleapis.com + - metadata.google.internal + - monitoring.googleapis.com +``` + +Next, we write a DNS policy for hipstershop that allows egress traffic to the trusted google sites. The following network policy allows egress access for all destination selectors labeled, `google`. Note that putting domains in a network set referencing it in policy is the best practice. Also, note that using `selector: all()` should only be used if all pods in the namespace can access all of the domains in the network set; if not, you should create separate policies accordingly. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: application.allow-egress-domain + namespace: hipstershop +spec: + tier: application + order: 0 + selector: all() + serviceAccountSelector: '' + egress: + - action: Allow + source: {} + destination: + selector: destinations == "google" + types: + - Egress +``` + +## Network sets in Service Graph + +Continuing with our `hipstershop` example, if you go to Service Graph, you see hipstershop (highlighted in yellow). + +![hipstershop](/img/calico-enterprise/hipstershop.png) + +If we double-click `hipstershop` to drill down, we now see the `google` network set icon (highlighted in yellow). We now have visibility to traffic external from google sites to hipstershop. (If you are using the tigera-labs cluster, note that the network set will not be displayed as shown below.) + +![google-networkset](/img/calico-enterprise/google-networkset.png) + +Service Graph provides a view into how services are interconnected in a consumable view, along with easy access to flow logs. You can also see traffic associated with network sets in volumetric display with Flow Visualizer. + +## Tutorial + +In the following example, we create a global network set resource for a trusted load-balancer that can be used with microservices and applications. The label, `trusted-ep: load-balancer` is how this global network set can be referenced in policy. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkSet +metadata: + name: load-balancer + labels: + trusted-ep: "load-balancer" +spec: + nets: + # Modify the ip addresses to refer to the ip addresses of load-balancers in your environment + - 10.0.0.1/32 + - 10.0.0.2/32 +``` + +The following network policy uses the `selector: trusted-ep == "load balancer"` to reference the above GlobalNetworkSet. All applications in the `app2-ns` namespace, that match `app2` and `svc1` are allowed ingress traffic from the trusted load balance on port 1001. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: application.app2-svc1 + namespace: app2-ns +spec: + tier: application + order: 500 + selector: (app == "app2"&&svc == "svc1") + ingress: + - action: Allow + protocol: TCP + source: + selector: trusted-ep == "load-balancer" + destination: + ports: + - '10001' + types: + - Ingress +``` + +### Advanced policy rules with network sets + +When you combine $[prodname] policy rules with network sets, you have powerful ways to fine-tune. The following example combines network sets with specific rules in a global network policy to deny access more quickly. +We start by creating a $[prodname] GlobalNetworkSet that specifies a list of CIDR ranges we want to deny: 192.0.2.55/32 and 203.0.113.0/24. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkSet +metadata: + name: ip-protect + labels: + ip-deny-list: 'true' +spec: + nets: + - 192.0.2.55/32 + - 203.0.113.0/24 +``` + +Next, we create two $[prodname] GlobalNetworkPolicy resources. The first is a high "order" policy that allows traffic as a default for things that don’t match our second policy, which is low "order" and uses the GlobalNetworkSet label as a selector to deny ingress traffic (IP-deny-list in the previous step). In the label selector, we also include the term, `!has(projectcalico.org/namespace)`, which prevents this policy from matching pods or NetworkSets that also have this label. To more quickly enforce the denial of forwarded traffic to the host at the packet level, use the `doNotTrack` and `applyOnForward` options. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: forward-default-allow +spec: + selector: apply-ip-protect == 'true' + order: 1000 + doNotTrack: true + applyOnForward: true + types: + - Ingress + ingress: + - action: Allow +--- +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: ip-protect +spec: + selector: apply-ip-protect == 'true' + order: 0 + doNotTrack: true + applyOnForward: true + types: + - Ingress + ingress: + - action: Deny + source: + selector: ip-deny-list == 'true' && !has(projectcalico.org/namespace) +``` + +## Best practices for using network sets + +- Create network sets as soon as possible after getting started + + This allows you to quickly realize the benefits of seeing custom metadata in flow logs and visualizing traffic in Service Graph and Flow Visualizer. + +- Create a network set label and name schema + + It is helpful to think: what names would be meaningful and easy to understand when you look in Service Graph or Flow Visualizer? What labels will be easy to understand when used in network policies – especially if you are separating users who manage network sets from those who consume them in network policies. + +- Do not put large sets of CIDRs and domains directly in policy + + Network sets allow you to specify CIDRs and/or domains. Although you can add CIDRs and domains directly in policy, it doesn't scale. + +- Do not put thousands of rules into a policy, each with a different CIDR + + If your set of /32s can be easily aggregated into a few broader CIDRs without compromising security, it’s a good thing to do; whether you’re putting the CIDRs in the rule or using a network set. + +- If you want to match thousands of endpoints, write one or two rules and use selectors to match the endpoints. + + Having one rule per port, per host is inefficient because each rule ends up being rendered as an iptables/eBPF rule instead of making good use of IP sets. + +- Avoid overlapping IP addresses/subnets in networkset/globalnetworkset definitions + +## Efficient use of network sets + +If you have a large number of things to match, using a network set is more efficient both in the control plane (for example, Felix CPU), and for the packet path (latency/per packet CPU). If you use network sets and you add/remove an IP from the network set, this doesn't require changing iptables rules at all. It only requires updating the ipset, which is efficient. If you also change the policy rules, then iptables must be updated too. Using network sets is efficient for all of the following use cases: + +- The system applying the iptables rules to incoming connections (to decide whether to allow or deny the traffic) +- iptables rules updates whenever one of the policies and/or network sets change +- The Kubernetes APIserver handling changes to the policy and/or networkset CRDs + +Follow these guidelines for efficient use of network sets. + +| Policy | Network set | Results | +| ------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| source: selector: foo="bar" | With handful of broad CIDRs | **Efficient**
**-** 1 iptables/eBPF rule
- 1 IP set with handful of CIDRs | +| source: nets: [ ... handful ...] | Not used | **Efficient**
- Handful of iptables/eBPF rules
- 0 IP sets | +| source: selector: foo="bar" | One network set with 2000 x /32s | **`*`Most efficient**
- 1 iptables/eBPF rule
- 1 IP sets with 2000 entries | +| | Two network sets with 1000 each x /32s | **Efficient**
- 2 iptables/eBPF rules
- 2 IP set with 1000 entries | +| source:
nets: [... 2000 /32s ...]
- source:
nets: [1 x /32]
- source: nets: [1 x /32]
- ... x 2000 | Not used | **Inefficient**
Results in programming 2k iptables/eBPF rules
- 2000+ iptables/eBPF rules
- 0 IP sets | + +`*` Updating **ipsets** is fast and efficient. Adding/removing a single entry is an O(1) operation no matter the number of IPs in the set. Updating **iptables** is generally slow and gets slower the more rules you have in total (including rules created by kube-proxy, for example). Less than 10K rules are generally fine, but noticeable latency occurs when updating rules above that number (increasing as the number of rules grows). (The newer nftables may scale more efficiently but those results are not included here.) + +Similarly, hitting many iptables (or eBPF) rules adds latency to the first packet in a flow. For iptables, it measures around 250ns per rule. Although a single rule is negligible, hitting 10K rules add >1ms to the first packet in the flow. Packets only hit the rules for the particular interface that they arrive on; if you have 10K rules on one interface and 10 rules on another, the packet processed by the first interface will have more latency. + +## Additional resources + +- [Network set](../reference/resources/networkset.mdx) +- [Global network set](../reference/resources/globalnetworkset.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/policy-best-practices.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-best-practices.mdx new file mode 100644 index 0000000000..87ebe87b5c --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-best-practices.mdx @@ -0,0 +1,355 @@ +--- +description: Best practices for Calico Cloud policy across connected clusters — security posture, scalability with tiers, and performance tuning under load. +--- + +# Policy best practices + +## Big picture + +Policy best practices for run-time security starts with $[prodname]’s robust network security policy, but other $[prodname] resources play equally important roles in security, scalability, and performance. + +Learn $[prodname] policy best practices and resources that support a zero trust network model: + +- [Prepare for policy authoring](#prepare-for-policy-authoring) +- [Policy best practices for day-one zero trust](#policy-best-practices-for-day-one-zero-trust) +- [Policy design for efficiency and performance](#policy-design-for-efficiency-and-performance) +- [Policy life cycle tools](#policy-life-cycle-tools) + +## Prepare for policy authoring + +### Determine who can write policy + +Any team familiar with deploying microservices in Kubernetes can easily master writing network policies. The challenge in many organizations is deciding who will be given permission to write policy across teams. Although there are different approaches, $[prodname] policy tools have the flexibility and guardrails to accommodate different approaches. + +Let’s review two common approaches. + +- **Microservices teams write policy** + + In this model, network policy is treated as code, built into and tested during the development process, just like any other critical part of a microservice’s code. The team responsible for developing a microservice has a good understanding of other microservices they consume and depend on, and which microservices consume their microservice. With a defined, standardized approach to policy and label schemas, there is no reason that the teams cannot implement network policies for their microservice as part of the development of the microservice. With visibility in Service Graph, teams can even do basic troubleshooting. + +- **Dev/Ops writes policy, microservice team focuses on internals** + An equally valid approach is to have development teams focus purely on the internals of the microservices they are responsible for, and leave responsibility for operating the microservices with DevOps teams. A Dev/ops team needs the same understanding as the microservices team above. However, network security may come much later in the organization’s processes, or even as an afterthought on a system already in production. This can be more challenging because getting network policies wrong can have significant production impacts. But using $[prodname] tools, this approach is still achievable. + +When you get clarity on who can write policies, you can move to creating tiers. $[prodname] tiers, along with standard Kubernetes RBAC, provide the infrastructure to meet security concerns across teams. + +### Understand the depth of $[prodname] network policy + +Because $[prodname] policy goes well beyond the features in Kubernetes policy, we recommend that you have a basic understanding of [network policy and global network policy](beginners/calico-network-policy.mdx) and how they provide workload access controls. And even though you may not implement the following policies, it is helpful to know the depth of defense that is available in $[prodname]. + +- [Policy for services](beginners/services/index.mdx) +- [Policy integration for firewalls](policy-firewalls/index.mdx) + +### Create policy tiers + +**Tiers** are a hierarchical construct used to group policies and enforce higher precedence policies that cannot be circumvented by other teams. As part of your microsegmentation strategy, tiers let you apply identity-based protection to workloads and hosts. + +Before creating policies, we recommend that you create your tier structure. This often requires internal debates and discussions. As noted previously, $[prodname] policy workflow has the guardrails you need to allow diverse teams to participate in policy writing. + +To understand how tiered policy works and best practices, see [Get started with tiered policies](policy-tiers/tiered-policy.mdx). + +### Create label standards + +Creating a label standard is often an overlooked step. But if you skip this step, it will cost you in troubleshooting down the road; especially given visibility/troubleshooting is already a challenge in a Kubernetes deployment. + +**Why are label standards important?** + +Network policies in Kubernetes depend on **labels and selectors** (not IP addresses and IP ranges) to determine which workloads can talk to each other. As pods dynamically scale up and down, network policy is enforced based on the labels and selectors that you define. So workloads and host endpoints need unique, identifiable labels. If you create duplicate label names, or labels are not intuitive, troubleshooting network policy issues and authoring network policies becomes more difficult. + +**Recommendations**: + +- Follow the [Kubernetes guidelines for labels](https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/). If the Kubernetes guidelines do not cover your use cases, we recommend this blog from Tigera Support: [Label standard and best practices for Kubernetes security](https://www.helpnetsecurity.com/2021/05/26/kubernetes-security/). +- Develop a comprehensive set of labels that meets the deployment, reporting, and security requirements of different stakeholders in your organization. +- Standardize the way you label your pods and write your network policies using a consistent schema or design pattern. +- Labels should be defined to achieve a specific and explicit purpose +- Use an intuitive language in your label definition that enables a quick and simple identification of labeled Kubernetes objects. +- Use label key prefixes and suffixes to identify attributes required for asset classification. +- Ensure the right labels are applied to Kubernetes objects by implementing label governance checks in your CI/CD pipeline or at runtime. + +### Create network sets + +Network sets and global network sets are grouping mechanisms for arbitrary sets of IPs/subnets/CIDRs or domains. They are key resources for efficient policy design. The key use cases for network sets are: + +- **Use/reuse in policy to support scaling** + + You reference network sets in policies using selectors (rather than updating individual policies with CIDRs or domains). + +- **Visibility to traffic to/from a cluster** + + For apps that integrate with third-party APIs and SaaS services, you get enhanced visibility to this traffic in Service Graph. + +- **Global deny lists** + + Create a “deny-list” of CIDRs for bad actors or embargoed countries in policy. + +**Recommendation**: Create network sets **and labels** before writing policy. + +For network set tutorial and best practices, see [Get started with network sets](networksets.mdx). + +## Policy best practices for day-one zero trust + +### Create a global default deny policy + +A global default deny network policy provides an enhanced security posture – so pods without policy (or incorrect policy) are not allowed traffic until appropriate network policy is defined. We recommend creating a global default deny, regardless of whether you use Calico Enterprise and/or Kubernetes network policy. + +But, be sure to understand the [best practices for creating a default deny policy](default-deny.mdx) to avoid breaking your cluster. + +Here are sample [default deny policies](beginners/kubernetes-default-deny.mdx). + +### Define both ingress and egress network policy rules for every pod in the cluster + +Although defining network policy for traffic external to clusters (north-south) is certainly important, it is equally important to defend against attacks for east-west traffic. Simply put, **every connection from/to every pod in every cluster should be protected**. Although having both doesn’t guarantee protection against other attacks and vulnerabilities, one innocuous workload can lead to exposure of your most critical workloads. + +For examples, see [basic ingress and egress policies](beginners/calico-network-policy.mdx). + +## Policy design for efficiency and performance + +Teams can write policies that work, but ultimately you want policies that also scale, and do not negatively impact performance. + +If you follow a few simple guidelines, you’ll be well on your way to writing efficient policy. + +### Use global network policy only when all rules apply globally + +- **Do** + + Use global network policy for cluster-wide scope when all rules apply to multiple namespaces or host endpoints. For example, use a global network policy to create a deny-list of CIDRs for embargoed countries, or for global default deny everywhere, even for new namespaces. + + Why? Although at the level of packet processing there is no difference between network policy and global network, for CPU usage, one global network policy is faster than a large number of network policies. + +- **Avoid** + + Using a global network policy as a way to combine diverse, namespaced endpoints with different connectivity requirements. Although creating such a policy can work, appears efficient and is easier to view than several separate network policies, it is inefficient and should be avoided. + + Why? Putting a lot of anything in policy (rules, CIDRs, ports) that are manipulated by selectors is inefficient. iptables/eBPF rules depend on minimizing executions and updates. When a selector is encountered in a policy rule, it is converted into one iptables rule that matches on an IP set. Then, different code keeps the IP sets up to date; this is more efficient than updating iptables rules. Also, because iptables rules execute sequentially in order, having many rules results in longer network latencies for the first packet in a flow (approximately 0.25-0.5us per rule). Finally, having more rules slows down programming of the data plane, making policy updates take longer. + +**Example: Inefficient global network policy** + +The following policy is a global network policy for a microservice that limits all egress communication external to the cluster in the security tier. Does this policy work? Yes. And logically, it seems to cleanly implement application controls. + +```yaml noValidation +1 apiVersion: projectcalico.org/v3 +2 kind: GlobalNetworkPolicy +3 metadata: +4 name: security.allow-egress-from-pods +5 spec: +6 tier: security +7 order: 1 +8 selector: all() +9 egress: +10 - action: Deny +11 source: +12 namespaceSelector: projectcalico.org/namespace starts with "tigera" +13 destination: +14 selector: threatfeed == "feodo" +15 - action: Allow +16 protocol: TCP +17 source: +18 namespaceSelector: kubernetes.io/metadata.name == "sso" +19 ports: +20 - '443' +21 - '80' +22 destination: +23 domains: +24 - '*.googleapis.com' +25 - action: Allow +26 protocol: TCP +27 source: +28 selector: psql == "external" +29 destination: +30 ports: +31 - '5432' +32 domains: +33 - '*.postgres.database.azure.com' +34 - action: Allow +35 protocol: TCP +36 source: {} +37 destination: +38 ports: +39 - '443' +40 - '80' +41 domains: +42 - '*.logic.azure.com' +43 - action: Allow +44 protocol: TCP +45 source: {} +46 destination: +47 ports: +48 - '443' +49 - '80' +50 domains: +51 - '*.azurewebsites.windows.net' +52 - action: Allow +53 protocol: TCP +54 source: +55 selector: 'app in { "call-archives-api" }||app in { "finwise" }' +56 destination: +57 domains: +58 - '*.documents.azure.com' +59 - action: Allow +60 protocol: TCP +61 source: +62 namespaceSelector: kubernetes.io/metadata.name == "warehouse" +63 destination: +64 ports: +65 - '1433' +66 domains: +67 - '*.database.windows.net' +68 - action: Allow +69 protocol: TCP +70 source: {} +71 destination: +72 nets: +73 - 65.132.216.26/32 +74. - 10.10.10.1/32 +75 ports: +76 - '80' +77 - '443' +78 - action: Allow +79 protocol: TCP +80 source: +81 selector: app == "api-caller" +82 destination: +83 ports: +84 - '80' +85 - '443' +86 domains: +87 - api.example.com +88 - action: Allow +89 source: +90 selector: component == "tunnel" +91 - action: Allow +92 destination: +93 selector: all() +94 namespaceSelector: all() +95 - action: Deny +96 types: +97 - Egress +``` + +**Why this policy is inefficient** + +First, the policy does not follow guidance on use for global network policy: that all rules apply to the endpoints. So the main issue is inefficiency, although the policy works. + +The main selector `all()` (line 8) means the policy will be rendered on every endpoint (workload and host endpoints). The selectors in each rule (for example, lines 12 and 14) control traffic that are matched by that rule. So, even if the host doesn’t have any workloads that match `"selector: app == "api-caller"`, you’ll still get the iptables/eBPF rule rendered on every host to implement that rule. If this sample policy had 100 pods, that’s a 10 - 100x increase in the number of rules (depending on how many local endpoints match each rule). In short, it adds: + +- Memory and CPU to keep track of all the extra rules +- Complexity to handle changes to endpoint labels, and to re-render all the policies too. + +### Avoid policies that may select unwanted endpoints + +The following policy is for an application in a single namespace, `app1-ns` namespace. There are two microservices that are all labeled appropriately: + +- microservice 1 has `app: app1`, `svc: svc1` +- microservice 2 has `app: app1`, `svc: svc2` + +The following policy works correctly and does not incur a huge performance hit. But it could select additional endpoints that were not intended. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: application.app1 + namespace: app1-ns +spec: + tier: application + order: 10 + selector: app == "app1" + types: + - Ingress + ingress: + - action: Allow + source: + selector: trusted-ip == "load-balancer" + destination: + selector: svc == "svc1" + ports: + - 10001 + protocol: TCP + - action: Allow + source: + selector: svc == "svc1" + destination: + selector: svc == "svc2" + ports: + - 10002 + protocol: TCP +``` + +The policy incorrectly assumes that the main policy selector (`app == "app1"`) will be combined (ANDed) with the endpoint selector, and only for certain policy types. In this case, + +- **Ingress** - combines policy selector and _destination endpoint selector_ + or +- **Egress** - combines policy selector and _source endpoint selector_ + +But if the assumptions behind the labels are not understood by other policy authors and are not correctly assigned, the endpoint selector may select _additional endpoints that were not intended_. For ingress policy, this can open up the endpoint to more IP addresses than necessary. This unintended consequence would be exacerbated if the author used a global network policy. + +### Put multiple relevant policy rules together in the same policy + +As discussed previously, it is better to create separate policies for different endpoint connectivity rules, than a single global network policy. However, you may interpret this to mean that the best practice is to make unique policies that do not aggregate any rules. But that is not the case. Why? When $[prodname] calculates and enforces policy, it updates the iptables/eBPF and reads policy changes and pod/workload endpoints from the datastore. The more policies in memory, the more work it takes determine which policies match a particular endpoint. If you group more rules into one policy, there are fewer policies to match against. + +### Understand effective use of label selectors + +Label selectors abstract network policy from the network. Misuse of selectors can slow things down. As discussed previously, the more selectors you create, the harder $[prodname] works to find matches. + +The following policy shows an inefficient use of selectors. Using `selector: all()` renders the policy on all nodes for all workloads. If there are 10,000 workloads, but only 10 match label==foo, that is very inefficient at the data plane level. + +```yaml +selector: all() +ingress: + - source: + selector: label == 'bar' + destination: + selector: label == 'foo' +``` + +The best practice policy below allows the same traffic, but is more efficient and scalable. Why? Because the policy will be rendered only on nodes with workloads that match the selector `label==foo`. + +```yaml +selector: label == 'foo' +ingress: + source: + selector: label == 'bar' +``` + +Another common mistake is using `selector: all()` when you don’t need to. `all()` means _all workloads_ so that will be a large IP set. Whenever there's a source/destination selector in a rule, it is rendered as an IP set in the data plane. + +```yaml +source: + selector: all() +``` + +### Put domains and CIDRs in network sets rather than policy + +Network sets allow you to specify CIDRs and/or domains. As noted in [Network set best practices](policy-best-practices.mdx), we do not recommend putting large CIDRs or domains directly in policy. Although nothing stops you from do this in policy, using network sets is more efficient and supports scaling. + +## Policy life cycle tools + +### Preview, stage, deploy + +A big obstacle to adopting Kubernetes is not having confidence that you can effectively prevent, detect, and mitigate across diverse teams. The following policy life cycle tools in the web console (**Policies** tab) can help. + +- **Policy recommendations** + + Get a policy recommendation for unprotected workloads. Speeds up learning, while supporting zero trust. + +- **Policy impact preview** + + Preview the impacts of policy changes before you apply them to avoid unintentionally exposing or blocking other network traffic. + +- **Policy staging and audit modes** + + Stage network policy so you can monitor traffic impact of both Kubernetes and $[prodname] policy as if it were actually enforced, but without changing traffic flow. This minimizes misconfiguration and potential network disruption. + +For details, see [Policy life cycle tools](staged-network-policies.mdx). + +### Do not trust anything + +Zero trust means that you do not trust anyone or anything. $[prodname] handles authentication on a per request basis. Every action is either authorized or restricted, and the default is everything is restricted. To apply zero trust to policy and reduce your attack surface and risk, we recommend the following: + +- Ensure that all expected and allowed network flows are explicitly allowed; any connection not explicitly allowed is denied + +- Create a quarantine policy that denies all traffic that you can quickly apply to workloads when you detect suspicious activity or threats + +## Additional resources + +- [Troubleshoot policies](policy-troubleshooting.mdx) +- [Security and policy best practices blog](https://www.tigera.io/blog/kubernetes-security-policy-10-critical-best-practices/) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/policy-firewalls/fortinet-integration/firewall-integration.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-firewalls/fortinet-integration/firewall-integration.mdx new file mode 100644 index 0000000000..93ef4f9a94 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-firewalls/fortinet-integration/firewall-integration.mdx @@ -0,0 +1,251 @@ +--- +description: Use a FortiGate firewall to control egress traffic from Kubernetes workloads in a Calico Cloud connected cluster. +--- + +# Extend Kubernetes to Fortinet firewall devices + +:::warning[deprecation and removal notice] + +The Fortinet integration is deprecated and will be removed in a future release. +If you want to use $[prodname] with Fortinet or any other firewall, we recommend instead using an [egress gateway](../../../networking/egress/egress-gateway-on-prem.mdx). + +::: + +## Big picture + +Use $[prodname] network policy to control traffic from Kubernetes clusters in your FortiGate firewalls. + +## Value + +As platform and security engineers, you want your apps to securely communicate with the external world. But you also want to secure the network traffic from the Kubernetes clusters using your FortiGate firewalls. Using the Fortinet/$[prodname] integration, security teams can retain firewall responsibility, secure traffic using $[prodname] network policy, which frees up time for ITOps. + +## Concepts + +## Integration at a glance + +This $[prodname]/Fortinet integration workflow lets you control egress traffic leaving the Kubernetes cluster. You create perimeter firewall policies in FortiManager and FortiGate that reference Kubernetes workloads. $[prodname] acts as a conduit, using the `tigera-firewall-controller` and global network policies to pass Kubernetes workload information to FortiManager and FortiGate devices where policies are applied and enforced. + +The basic workflow is: + +1. Determine the Kubernetes pods that are allowed access outside the perimeter firewall. +1. Create $[prodname] global network policies with selectors that match those pods. Each global network policy maps to an address group in the FortiGate firewall. +1. Deploy the `tigera firewall controller` in the Kubernetes cluster. +1. Create a ConfigMap with Fortinet firewall information. + The `tigera firewall controller` reads the ConfigMap, gets the FortiGate firewall IP address, API token, and source IP address selection with `node` or `pod`. In your Kubernetes cluster, the controller populates pod IPs or Kubernetes node IPs of selector matching pods in FortiGate address group objects. + +## Before you begin + +**Supported versions** + +- FortiGate v6.2 +- FortiManager v6.4 + +**Required** + +{/*//TODO-CC-XREFS - Pull secret that you used during [$[prodname] installation](/getting-started/install-on-clusters/index)*/} +- IPv4 CIDR’s or IP addresses of all Kubernetes nodes; this is required for FortiManager to treat Kubernetes nodes as trusted hosts. + +**Recommended** + +- Experience creating and administering FortiGate/FortiManager firewall policies +- Experience using [$[prodname] tiers](../../../reference/resources/tier.mdx) and [Global network policy](../../../reference/resources/globalnetworkpolicy.mdx) + +## How to + +- [Create tier and global network policy](#create-tier-and-global-network-policy) +- [Configure FortiGate firewall to communicate with firewall controller](#configure-fortigate-firewall-to-communicate-with-firewall-controller) +- [Configure FortiManager to communicate with firewall controller](#configure-fortimanager-to-communicate-with-firewall-controller) +- [Create a config map for address selection in firewall controller](#create-a-config-map-for-address-selection-in-firewall-controller) +- [Create a config map with FortiGate and FortiManager information](#create-a-config-map-with-fortigate-and-fortimanager-information) +- [Install FortiGate ApiKey and FortiManager password as secrets](#install-fortigate-apikey-and-fortimanager-password-as-secrets) +- [Deploy firewall controller in the Kubernetes cluster](#deploy-firewall-controller-in-the-kubernetes-cluster) + +### Create tier and global network policy + +1. Create a tier for organizing global network policies. + + Create a new [Tier](../../policy-tiers/tiered-policy.mdx) to organize all FortiGate firewall global network policies in a single location. + +1. Note the tier name to use in a later step for the FortiGate firewall information config map. + +1. Create a GlobalNetworkPolicy for address group mappings. + + For example, a GlobalNetworkPolicy can select a set of pods that require egress access to external workloads. In the following GlobalNetworkPolicy, the firewall controller creates an address group named, `default.production-microservice1` in the FortiGate firewall. The members of `default.production-microservice1` address group include IP addresses of nodes. Each node can host one or more pods whose label selector match with `env == 'prod' && role == 'microservice1'`. Each GlobalNetworkPolicy maps to an address group in FortiGate firewall. + + ```yaml + apiVersion: projectcalico.org/v3 + kind: GlobalNetworkPolicy + metadata: + name: default.production-microservice1 + spec: + selector: "env == 'prod' && role == 'microservice1'" + types: + - Egress + egress: + - action: Allow + ``` + +### Configure FortiGate firewall to communicate with firewall controller + +1. Determine and note the CIDR's or IP addresses of all Kubernetes nodes that can run the `tigera-firewall-controller`. + Required to explicitly allow the `tigera-firewall-controller` to access the FortiGate API. +1. Create an Admin profile with read-write access to Address and Address Group Objects. + For example: `tigera_api_user_profile` +1. Create a REST API Administrator, associate this user with the `tigera_api_user_profile` profile, and add the CIDR or IP address of your Kubernetes cluster nodes as trusted hosts. + For example: `calico_enterprise_api_user` +1. Note the API key. + +### Configure FortiManager to communicate with firewall controller + +1. Determine and note the CIDR's or IP addresses of all Kubernetes nodes that can run the `tigera-firewall-controller`. + Required to explicitly allow the tigera-firewall-controller to access the FortiManager API. +1. From system settings, create an Admin profile with Read-Write access for `Policy & Objects`. + For example: `tigera_api_user_profile` +1. Create a JSON API administrator, associate this user with the `tigera_api_user_profile` profile, and add the CIDR or IP address of your Kubernetes cluster nodes as `Trusted Hosts`. +1. Note the username and password. + +### Create a config map for address selection in firewall controller + +1. Create a namespace for tigera-firewall-controller. + + ```bash + kubectl create namespace tigera-firewall-controller + ``` + +1. Create a config map with FortiGate firewall information. + + For example: + + ```bash + kubectl -n tigera-firewall-controller create configmap tigera-firewall-controller \ + --from-literal=tigera.firewall.policy.selector="projectcalico.org/tier == 'default'" \ + --from-literal=tigera.firewall.addressSelection="node" + ``` + + **ConfigMap values** + + | Field | Enter values... | + | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | + | tigera.firewall.policy.selector | The tier name with the global network policies with the FortiGate address group mappings.
For example, this selects the global network policies in the `default` tier:
`tigera.firewall.policy.selector: "projectcalico.org/tier == 'default'" | + | tigera.firewall.addressSelection | The addressSelection for outbound traffic leaving the cluster.
For example, if outgoingNat is enabled in cluster and compute Node IP address is used "tigera.firewall.addressSelection == `node` or
If pod IP address used then "tigera.firewall.addressSelection == `pod`" | + +### Create a config map with FortiGate and FortiManager information + +1. In the [FortiGate ConfigMap manifest]($[filesUrl_CE]/manifests/fortinet-device-configmap.yaml), add your FortiGate firewall information in the data section, `tigera.firewall.fortigate`. + + Where: + + | Field | Description | + | ------------------------ | --------------------------------------------------------------------------- | + | name | FortiGate device name | + | ip | FortiGate Management Ip address | + | apikey | Secret in tigera-firewall-controller namespace, to store FortiGate's APIKey | + | apikey.secretKeyRef.name | Name of the secret to store APIKey. | + | apikey.secretKeyRef.key | Key name in the secret, which stores APIKey | + + For example: + + ```yaml + - name: prod-eastcoast-1 + ip: 1.2.3.1 + apikey: + secretKeyRef: + name: fortigate-east1 + key: apikey-fortigate-east1 + - name: prod-eastcoast-2 + ip: 1.2.3.2 + apikey: + secretKeyRef: + name: fortigate-east2 + key: apikey-fortigate-east2 + ``` + +1. In the [FortiManager ConfigMap manifest]($[filesUrl_CE]/manifests/fortinet-device-configmap.yaml), add your FortiManager information in the data section, `tigera.firewall.fortimgr`. + + Where: + + | Field | Description | + | -------------------------- | ------------------------------------------------------------------------------ | + | name | FortiManager device name | + | ip | FortiManager Management Ip address | + | adom | FortiManager ADOM name to manage kubernetes cluster. | + | username | JSON api access account name to Read/Write FortiManager address objects. | + | password | Secret in tigera-firewall-controller namespace, to store FortiManager password | + | password.secretKeyRef.name | Name of the secret to store password. | + | password.secretKeyRef.key | Key name in the secret, which stores password. | + + For example: + + ```yaml + - name: prod-east1 + ip: 1.2.4.1 + username: api_user + adom: root + password: + secretKeyRef: + name: fortimgr-east1 + key: pwd-fortimgr-east1 + ``` + +:::note + +If you are not using FortiManager in the integration, include only the following field in the ConfigMap data section. `tigera.firewall.fortimgr: |` + +::: + +1. Apply the manifest. + + ``` + kubectl apply -f $[filesUrl_CE]/manifests/fortinet-device-configmap.yaml + ``` + +### Install FortiGate ApiKey and FortiManager password as secrets + +1. Store each FortiGate API key as a secret in the `tigera-firewall-controller` namespace. + For example, the FortiGate device, `prod-east1`, store its ApiKey as a secret name as `fortigate-east1`, with key as `apikey-fortigate-east1`. + + ``` + kubectl create secret generic fortigate-east1 \ + -n tigera-firewall-controller \ + --from-literal=apikey-fortigate-east1= + ``` + +1. Store each FortiManager password as secret in the `tigera-firewall-controller` namespace. + For example, for FortiMgr `prod-east1`, store its password as a secret name as `fortimgr-east1`, with key as `pwd-fortimgr-east1`. + + ``` + kubectl create secret generic fortimgr-east1 \ + -n tigera-firewall-controller \ + --from-literal=pwd-fortimgr-east1= + ``` + +### Deploy firewall controller in the Kubernetes cluster + +1. Install your pull secret. + + ``` + kubectl create secret generic tigera-pull-secret \ + --from-file=.dockerconfigjson= \ + --type=kubernetes.io/dockerconfigjson -n tigera-firewall-controller + ``` + +1. Apply the manifest. + + ``` + kubectl apply -f $[filesUrl_CE]/manifests/fortinet.yaml + ``` + +## Verify the integration + +1. Log in to the FortiGate firewall user interface. +1. Under **Policy & Objects**, click **Addresses**. +1. Verify that your Kubernetes-related address objects and address group objects are created with the following comments "Managed by Tigera $[prodname]". + +For all FortiManagers that are configured to work with firewall-controller, log in to each FortiManager UI with the correct ADOM. + +1. Click **Policy & Objects**, **Object Configuration**, \*\*Addresses. +1. Verify that your Kubernetes-related address objects and address group objects are created with the following comments "Managed by Tigera $[prodname]". + +## Additional resources + +- [Extend FortiManager firewall policies to Kubernetes](fortimgr-integration.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/policy-firewalls/fortinet-integration/fortimgr-integration.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-firewalls/fortinet-integration/fortimgr-integration.mdx new file mode 100644 index 0000000000..8a724895ca --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-firewalls/fortinet-integration/fortimgr-integration.mdx @@ -0,0 +1,164 @@ +--- +description: Extend FortiManager firewall policies into Kubernetes workloads in a Calico Cloud connected cluster. +--- + +# Extend FortiManager firewall policies to Kubernetes + +:::warning[deprecation and removal notice] + +The Fortinet integration is deprecated and will be removed in a future release. +If you want to use $[prodname] with Fortinet or any other firewall, we recommend instead using an [egress gateway](../../../networking/egress/egress-gateway-on-prem.mdx). + +::: + +## Big picture + +Use FortiManager firewall policies to secure workloads in your Kubernetes cluster. + +## Value + +The $[prodname]/Fortinet integration lets you control Kubernetes clusters directly and apply policy +using the FortiManager UI as the primary interface. This allows firewall administrators to leverage existing +tools and workflows as they learn and adopt Kubernetes orchestration at their own pace. + +## Concepts + +### Integration at a glance + +This $[prodname]/Fortinet solution lets you directly control Kubernetes policies using FortiManager. + +The basic workflow is: + +1. Determine the Kubernetes pods that you want to securely communicate with each other. +1. Label these pods using a key-value pair where key is the `tigera.io/address-group`, and value is the pod matching a label name. +1. In the FortiManager, select the cluster’s ADOM, and create an address group using the key-value pair associated with the pods. +1. Create firewall policies using the address groups for IPv4 Source address and IPv4 Destination Address, and select services and actions as you normally would to allow or deny the traffic. Under the covers, the $[prodname] integration controller periodically reads the FortiManager firewall policies for your Kubernetes cluster, converts them to $[prodname] global network policies, and applies them to clusters. +1. Use the $[prodname] web console to verify the integration, and then FortiManager UI to make all updates to policy rules. + +:::note + +The default value for reading FortiManager firewall policies is three seconds. To change the value, modify environment variable FW_FORTIMGR_EW_POLL_INTERVAL in FortiManager integration manifest; units are in seconds. + +::: + +## Before you begin + +**Supported version** + +- FortiManager v6.4 + +**Required** + +- IPv4 CIDR’s or IP addresses of all Kubernetes nodes; this is required for FortiManager to treat Kubernetes nodes as trusted hosts. + +{/*//TODO-CC-XREF +- Pull secret that you used during [$[prodname] installation](/getting-started/install-on-clusters/index) +- Login access to [the $[prodname] web console](/operations/cnx/access-the-manager/) +*/} + +**Recommended** + +- Experience with [ tiered policy](../../policy-tiers/tiered-policy.mdx) and [ global network policy](../../../reference/resources/globalnetworkpolicy.mdx) +- Experience creating and administering FortiGate/FortiManager firewall policies + +## How to + +- [Create a tier](#create-a-tier) +- [Configure FortiManager to communicate with firewall controller](#configure-fortimanager-to-communicate-with-firewall-controller) +- [Create a FortiManager config map](#create-a-fortimanager-config-map) +- [Install FortiManager password as secrets](#install-fortimanager-password-as-secrets) +- [Deploy the firewall controller in the Kubernetes cluster](#deploy-the-firewall-controller-in-the-kubernetes-cluster) +- [Verify the integration](#verify-the-integration) + +### Create a tier + +Create a [$[prodname] tier](../../policy-tiers/tiered-policy.mdx) in the $[prodname] web console for each Kubernetes cluster you want to secure. We recommend that you create a new tier (rather than reusing an existing tier) for all global network policies created by the $[prodname] integration controller. + +## Configure FortiManager to communicate with firewall controller + +1. Determine and note the CIDR’s or IP addresses of all Kubernetes nodes that can run the `tigera-firewall-controller`. + This is required to explicitly allow the `tigera-firewall-controller` to access the FortiManager API. +1. From system settings, create an Admin profile with Read-Write access for `Policy & Objects`. + For example: `tigera_api_user_profile` +1. Create a JSON API administrator and associate this user with the `tigera_api_user_profile` profile and add CIDR or IP address of your Kubernetes cluster nodes as `trusted hosts`. +1. Note the username and password. + +## Create a FortiManager config map + +1. Create a namespace for the tigera-firewall-controller. + + ```bash + kubectl create namespace tigera-firewall-controller + ``` + +1. In this [FortiManager ConfigMap manifest]($[filesUrl_CE]/manifests/fortimanager-device-configmap.yaml), add your FortiManager device information in the data section: `tigera.firewall.fortimanager-policies`. For example: + + ```yaml noValidation + tigera.firewall.fortimanager-policies: | + - name: prod-east1 + ip: 3.2.1.4 + username: api_user + adom: root + tier: + packagename: sacramento + password: + secretKeyRef: + name: fortimgr-east1 + key: pwd-fortimgr-east1 + ``` + + Where: + + | Field | Description | + | -------------------------- | ---------------------------------------------------------------------------------------------------------------- | + | name | FortiManager device name. | + | ip | FortiManager Management IP address. | + | adom | FortiManager ADOM name that manages Kubernetes cluster. | + | packagename | FortiManager Firewall package. All firewall rules targeted for Kubernetes cluster are packed under this package. | + | username | JSON api access account name to Read/Write FortiManager address objects. | + | password | Secret in tigera-firewall-controller namespace, to store FortiManager password | + | tier | Tier name you created in the $[prodname] web console | + | password.secretKeyRef.name | Name of the secret to store password. | + | password.secretKeyRef.key | Key name in the secret, which stores password. | + +1. Apply the manifest. + + ```bash + kubectl apply -f $[filesUrl_CE]/manifests/fortimanager-device-configmap.yaml + ``` + +## Install FortiManager password as secrets + +Store each FortiManager password as a secret in the `tigera-firewall-controller` namespace. + +For example, in the ConfigMap for FortiMgr `prod-east1`, store its password as a secret name as `fortimgr-east1`, with key as `pwd-fortimgr-east1`. + +```bash +kubectl create secret generic fortimgr-east1 \ +-n tigera-firewall-controller \ +--from-literal=pwd-fortimgr-east1= +``` + +### Deploy the firewall controller in the Kubernetes cluster + +1. Install your pull secret. + + ```bash + kubectl create secret generic tigera-pull-secret \ + --from-file=.dockerconfigjson= \ + --type=kubernetes.io/dockerconfigjson -n tigera-firewall-controller + ``` + +1. Apply the manifest. + + ```bash + kubectl apply -f $[filesUrl_CE]/manifests/fortimanager.yaml + ``` + +## Verify the integration + +1. Log in to FortiManager with the correct ADOM. +2. Select **Policy & Objects**, **Object Configuration**, and create new **Address Groups**. +3. Click **Policy packages** and select the Package assigned to your Kubernetes cluster. +4. Create a test firewall policy with the following fields: Name, IPv4 Source Address, IPv4 Destination Address, Service and Action. +5. Log in to the $[prodname] web console, and under the tier that you specified in the ConfigMap, verify that the GlobalNetworkPolicies are created. diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/policy-firewalls/fortinet-integration/index.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-firewalls/fortinet-integration/index.mdx new file mode 100644 index 0000000000..4f0a0da5e8 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-firewalls/fortinet-integration/index.mdx @@ -0,0 +1,18 @@ +--- +description: Calico Cloud integrations with Fortinet firewalls — FortiGate for traffic enforcement and FortiManager for policy management. +hide_table_of_contents: true +--- + +# Fortinet firewall integrations + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + +:::warning[deprecation and removal notice] + +The Fortinet integration is deprecated and will be removed in a future release. +If you want to use $[prodname] with Fortinet or any other firewall, we recommend instead using an [egress gateway](../../../networking/egress/egress-gateway-on-prem.mdx). + +::: + + diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/policy-firewalls/fortinet-integration/overview.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-firewalls/fortinet-integration/overview.mdx new file mode 100644 index 0000000000..60af00df8a --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-firewalls/fortinet-integration/overview.mdx @@ -0,0 +1,38 @@ +--- +description: Integrate Kubernetes clusters with existing Fortinet firewall workflows using Calico Cloud — architecture, components, and what each side enforces. +--- + +# Determine the best Calico Cloud/Fortinet solution + +## Big picture + +Determine the best $[prodname]/Fortinet solution to integrate Kubernetes clusters with your existing Fortinet firewall workflows. + +## Value + +Many security teams must work within the confines of their existing IT security architecture, even though perimeter firewalls do not meet the needs of Kubernetes clusters. The $[prodname]/Fortinet integration allows firewall administrators to leverage existing Fortinet security tools and workflows, continue meeting compliance requirements, while adopting Kubernetes orchestration using $[prodname] at their own pace. + +### Concepts + +The $[prodname]/Fortinet integration provides the following solutions. You can you use them separately, or together without contention. + +### Solution 1: Extend Kubernetes to Fortinet firewall devices + +**Use case**: Control egress traffic for Kubernetes clusters. + +**Problem**: Perimeter firewalls do not have the necessary information to act on traffic that leaves the cluster for Kubernetes workloads. + +**Solution**: The $[prodname]/Fortinet integration leverages the power of $[prodname] policy selectors to provide Kubernetes workload information to FortiManager and FortiGate devices. You create perimeter firewall policies in FortiManager and FortiGate that reference Kubernetes workloads. Policies are applied and enforced by FortiGate devices. And Firewall administrators can write cluster egress policies that reference Kubernetes workloads directly in Fortinet devices. + +### Solution 2: Extend FortiManager firewall policies to Kubernetes + +**Use case**: Control Kubernetes clusters directly and apply policy. + +**Problem**: To avoid disruption, teams need to leverage existing FortiManager as the primary user interface. + +**Solution**: Use FortiManager to create firewall policies that are applied as $[prodname] network policies on Kubernetes workloads. Use the power of a $[prodname] “higher-order tier” so Kubernetes policy is evaluated early in the policy processing order, but update policy using FortiManager UI. Use the $[prodname] web console as a secondary interface to verify the integration and troubleshoot using logs. + +## Next steps + +- [Extend Kubernetes to Fortinet firewall devices](firewall-integration.mdx) +- [Extend FortiManager firewall policies to Kubernetes](fortimgr-integration.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/policy-firewalls/index.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-firewalls/index.mdx new file mode 100644 index 0000000000..1dd4313c62 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-firewalls/index.mdx @@ -0,0 +1,11 @@ +--- +description: Integrate Calico Cloud policy with existing perimeter firewalls — extend rule scope from Kubernetes workloads out to the network edge. +hide_table_of_contents: true +--- + +# Policy for firewalls + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/policy-tiers/calico-system.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-tiers/calico-system.mdx new file mode 100644 index 0000000000..330de170af --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-tiers/calico-system.mdx @@ -0,0 +1,75 @@ +--- +description: Customize the behavior of the calico-system tier that Calico Cloud installs by default to keep its own components reachable. +--- + +# Change calico-system tier behavior + +:::warning +The `calico-system` tier contains policies that secure $[prodname] components and is critical to cluster integrity. It is controlled by the Tigera Operator, and policies in the tier should not be edited, and the tier should not be moved. Although you can change the behavior of calico-system using adjacent tiers, you can inadvertently break critical cluster traffic. We highly recommend that you work with Support to implement changes around `calico-system` to prevent service disruption. + +::: + +## Big picture + +Change traffic behavior of the tier that secures $[prodname] components. + +## Value + +Although the tier that secures $[prodname] components cannot be changed, you can create policies in adjacent tiers to change its behavior. + +## Concepts + +$[prodname] automatically creates the `calico-system` tier during installation with network policies that select traffic to and from Tigera components. These policies ensure that traffic required for $[prodname] operation is allowed, and that any unnecessary traffic involving Tigera components is denied. This tier prevents disruption of $[prodname] functionality in case of network policy misconfiguration impacting Tigera components, and denies unexpected traffic in case of defect or compromise. + +### Ownership and management of calico-system + +Tigera defines the `calico-system` tier and manages the policies within it. The Tigera Operator installs and monitors these policies, ensuring they always match the state defined by Tigera. Management by the Operator also ensures integrity for upgrades. + +:::note + +The `calico-system` tier and its policies should not be edited, and the tier should not be moved. However, if you inadvertently make changes they are automatically reverted by the Operator to ensure your cluster is always protected. + +::: + +## Tutorial + +### Change behavior of calico-system + +If you want to change the way traffic is enforced by the `calico-system` tier, you must create policy in an adjacent tier to meet your needs. For example, if a policy in the `calico-system` tier allows or denies traffic, and you want to change how that traffic is enforced, you can create a policy in a tier before `calico-system` that selects the same traffic to make your desired changes. Similarly, if a policy in the `calico-system` tier passes or does not select traffic that you want to enforce, you can create a policy in a tier after `calico-system` to select this traffic to meet the desired behavior. + +### Example: use preceding tier to tighten security + +Let's say a `calico-system` policy allows ingress traffic from a $[prodname] component that you do not use, and you want to tighten enforcement to not allow this traffic. + +Within a tier that comes before `calico-system`, you can create a policy that selects the same endpoint and contains ingress rules that deny traffic from that component and pass to `calico-system` for traffic from other components. + +```yaml + # calico-system.es-gateway-access allows ingress from deep packet inspection, a feature not utilized for the purpose of this example. + # This policy tightens the scope of allowed ingress to es-gateway without modifying the calico-system policy directly. + +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: preceding-tier.es-gateway-access + namespace: tigera-elasticsearch +spec: + # Place in a tier prior to calico-system. + tier: preceding-tier + + # Select the same endpoint as the original policy. + selector: k8s-app == 'tigera-secure-es-gateway' + ingress: + # Select the same component ingress. + - source: + selector: k8s-app == 'tigera-dpi' + namespaceSelector: name == 'tigera-dpi' + # Enact different behavior (originally: Allow) + action: Deny + + # Defer to calico-system for other ingress/egress decisions for this endpoint. + - action: Pass +``` + +This example shows how you can change the impact of the `calico-system` tier on traffic without modifying the tier itself. This makes your changes more maintainable, and allows the calico-system tier to continue to receive updates as $[prodname] evolves without you needing to reconcile your changes each release. + +For help to manage or change the behavior of the `calico-system` tier, contact Tigera Support. diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/policy-tiers/index.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-tiers/index.mdx new file mode 100644 index 0000000000..205530017e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-tiers/index.mdx @@ -0,0 +1,11 @@ +--- +description: Use Calico Cloud policy tiers to let platform, security, and app teams author and order policy independently across connected clusters. +hide_table_of_contents: true +--- + +# Policy tiers + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/policy-tiers/policy-tutorial-ui.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-tiers/policy-tutorial-ui.mdx new file mode 100644 index 0000000000..ff35297765 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-tiers/policy-tutorial-ui.mdx @@ -0,0 +1,246 @@ +--- +description: Tutorial for the Calico Cloud policy management UI — author, order, and stage policies inside tiers from the web console. +--- + +import Screenshot from '/src/___new___/components/Screenshot'; + +# Network policy tutorial + +## What you will learn: + +- How to create a policy in the web console +- How labels and selectors work +- Basics of policy ordering and tiers + +## Scenario + +Let's start with a sample Kubernetes cluster. + +![policy-tutorial-overview](/img/calico-enterprise/policy-tutorial-overview.png) + +| Item | Description | +| ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Kubernetes cluster | A Kubernetes cluster with four namespaces and three nodes that run the pods in the cluster. | +| Namespace | Four namespaces named blue, red, green, and purple represent different applications running in the cluster. | +| Pod | Pods with meaningful labels for our applications:
- FE (frontend pods)
- BE (backend pods) | +| NetworkSet | An arbitrary set of IP subnetworks/CIDRs or domains that can be matched by standard label selectors in Kubernetes or $[prodname] network policy. Network sets are a $[prodname] namespaced resource. | +| GlobalNetwork Set | An arbitrary set of IP subnetworks/CIDRs or domains that can be matched by standard label selectors in Kubernetes or $[prodname] network policy. Global network sets are a $[prodname] global resource. | +| ServiceAccount | Provides an identity for processes that run in a pod. Service accounts are a Kubernetes namespaced resource. | +| HostEndpoint (HEP) | Physical or virtual interfaces attached to a host that runs $[prodname]. HEPs enforce $[prodname] policy on the traffic that enters or leaves the host’s default network namespace through the interfaces. HEPs are a $[prodname] global resource. | +| External component | A machine (physical or virtual) that runs outside of the Kubernetes cluster. | + +## Create a network policy + +To follow along in the web console, click **Policies**. + +There are three main parts to every $[prodname] policy: + +- **Scope** - namespace or global +- **Applies to** - objects within the above scope to which policy rules will be applied using labels and selectors +- **Rules** - whether this policy affects ingress, egress, or both + - Ingress - policy rules to apply to connections inbound to the selected objects + - Egress - policy rules to apply to connections outbound from the selected objects + + + +Let's look at each part. + +## Scope + +Scope defines the reach of your policy. Use this dropdown to determine whether your policy applies globally or to a specific namespace. Think of scope as the "top-level scope" that can be further specified using the "Applies to" selection that follows. + +- **Global** + + If you select global, but do not add entries in the **Applies to** field to further limit scope, _every pod and host endpoint (HEP) in our cluster would be in scope_. The following example uses the global option to limit the scope to all pods and HEPs (noted by check marks). + + ![policy-tutorial-global-scope](/img/calico-enterprise/policy-tutorial-global-scope.png) + +- **Namespace** + + If you select namespace, but do not add entries in the **Applies to** field to further limit scope, _every pod in this policy's namespace would be in scope_. The following example uses the namespace option to limit the scope to pods in the RED namespace. + + ![policy-tutorial-namespace-scope](/img/calico-enterprise/policy-tutorial-namespace-scope.png) + +### Applies to + +As discussed above, selecting **Applies to** lets you further limit pods in a policy. You can think of it as the "top-level endpoint selector". You define labels on your endpoints, namespaces, and service accounts, then use label selectors to limit connections by matching the following object types: + +- **Endpoints** + + Specify one or more label selectors to match specific endpoints, or select all endpoints + +- **Namespaces** (available only when the Scope is global) + + Specify one or more label selectors to match specific namespaces, or select all namespaces + +- **Service Accounts** + + Specify or more label selectors to match specific service accounts, or select all service accounts + +For example, if we select the BLUE namespace and apply it to only pods with the label, `app/tier == FE`, + + + +the resulting scope in our diagram would be only the pods labeled, `FE`: + +![blue-namespace-pods](/img/calico-enterprise/blue-namespace-pods.png) + +### Rules + +In the Rules section, you specify whether the policy impacts ingress, egress, or both. + +Note that ingress and egress are defined from the point of view of the _scoped endpoints_ (pods or host endpoints). In the previous diagram, the scoped endpoints are the three pods labeled, `app/tier:FE`. + +- Ingress rules filter traffic _coming to_ the scoped endpoints +- Egress rules filter traffic _leaving_ the scoped endpoints + +Click **+ Add ingress rule** to access the **Rules** panel. + +### Endpoint selector + +The endpoint selector lets you select the endpoint traffic that is matched within the scope you've defined in the policy. + +In our example, the policy is scoped to endpoints that have the `app/tier == FE` label in the BLUE namespace. In the context of an egress rule, when we add the `app/tier == BE` endpoint selector, all TCP traffic from endpoints that have the`app/tier == BE` label will be allowed to the `app/tier == FE` endpoints. + + + +Note that endpoints that have the `app/tier == BE` label in other namespaces are not matched because the policy is namespace scoped. + +### Namespace selector + +This is where things can get interesting. In the previous example, we did not select anything in the namespace selector. Let's change the namespace selector to have both the BLUE and GREEN namespaces. + + + +Although the overall policy is scoped to the BLUE namespace, we can match endpoints in other namespaces on a per-rule basis. Note that the top-level scope that you select remains unchanged, meaning that the policy is still applied only to endpoints in the BLUE namespace. + +![namespace-selector](/img/calico-enterprise/namespace-selector.png) + +### Network selector + +Using the Nets selector, we can add CIDR addresses to be matched by the policy rule. + + + +### Service account selector + +Network policies can be also applied to the endpoint’s service account. + +Using the service account selector, we can apply rules to traffic from any endpoint whose service account matches the name or label selector. + + + +### Use Match All for wider matches in policy rules + +The **Match All** policy rule (`all()` in YAML) matches traffic for: + +- All endpoints in a namespace (if the policy scope is namespace) +- All endpoints (if the policy scope is global) + +Let's look at an example of using **Match All** traffic in a namespaced policy: + +- Scope is namespaced (BLUE) +- Applies to `app/tier == FE` + +Suppose we want to match traffic to the pod labeled `BE`, and the $[prodname] `networkset-1`. + +![match-all-namespace](/img/calico-enterprise/match-all-namespace.png) + +To do this, we can use the policy rule endpoint selector, **Match All**. + + + +Not only is the pod labeled `BE` included, but also the $[prodname] `networkset-1`. + +![match-all-endpoints-example](/img/calico-enterprise/match-all-endpoints-example.png) + +Note that we could have created individual selectors to match pods labeled, `BE` and for the `network-set-1`. + +**Match All traffic with namespace selectors** + +In the following example, if we select **Match All** endpoints, but in the **Namespace selector**, we select both the BLUE and GREEN namespaces, the results for matching are: all pods and network sets in the BLUE and GREEN namespaces. + +![namespace-match-all](/img/calico-enterprise/namespace-match-all.png) + +**Global selector** + +Let's see what happens when we select the **Global** selector. + + + +In our example, the Global selector selects HEPs and global network sets are selected. You might think that Global (`global()` in YAML) would select all endpoints, but it doesn't. Global means "do not select any namespaced resources" (which includes namespaced network set resources). Another way to express it is, do not select any workload endpoints. + +![heps-networksets](/img/calico-enterprise/heps-networksets.png) + +**Endpoint selector, unspecified** + +Next, let's see what happens when the policy rule does not specify any selection criteria. In this example, the rule selects all workloads, network sets, endpoints, and host endpoints within scope of the policy, including external components (the VM database). + +![unspecified](/img/calico-enterprise/unspecified.png) + +Now that you know the basic elements of a network policy, let's move on to policy ordering and tiers. + +## Policy ordering + +$[prodname] policies can have order values that control the order of precedence. For both network policies and global network policies, $[prodname] applies the policy with the lowest value first. + +![policy-ordering](/img/calico-enterprise/policy-ordering.png) + +### Mixing Kubernetes and $[prodname] policies + +Kubernetes and $[prodname] policies work side by side without a problem. However, Kubernetes network policies cannot assign an order value, so $[prodname] will set an implicit order value of 1000 to any Kubernetes network policies. + +:::note +Policies are immediately applied to any new connections. However, for existing connections that are already open, the policy changes will only take effect after the connection has been reestablished. This means that any ongoing sessions may not immediately reflect policy changes until they are initiated again. +::: + +### $[prodname] policies with no order value + +$[prodname] policies with order values take precedence. Policies without order values take lesser precedence and are processed alphabetically. + +## Tiers + +Tiers are a hierarchical construct used to group policies and enforce higher precedence policies that cannot be circumvented by other teams. Access to tiers is controlled using user role permissions. For example, a security team can implement high-level policy (for example, blocking access to/from IP ranges in particular countries), while developers in a later tier can control specific rules for the microservices of an app running in the cluster. + +### Policy processing overview + +When a new connection is processed by $[prodname], each tier that contains a policy that selects the endpoint processes the connection. Tiers are sorted by their order - the smallest number first. Policies in each tier are then processed in order from lowest to highest. For example, a policy of 800 is ordered before a policy of order 1000. + +- If a network policy or global network policy in the tier allows or denies the connection, then evaluation is done: the connection is handled accordingly. + +- If a network policy or global network policy in the tier passes the connection, the next tier containing a policy that selects the endpoint processes the connection + +After a Pass action, if no subsequent tier contains any policies that apply to the pod, the connection is allowed. + +If the tier contains policies that apply to the endpoint, but the policies take no action on the connection, the connection is dropped by an implicit deny. + +If no tiers contain policies that apply to the endpoint, the connection is allowed by an implicit allow. + +### Policies with no order value + +You can create policies without an order value. When a policy with no order value is placed in a tier with other policies that do have an order value, the policies are processed as follows: + +- Policies are evaluated from smallest to largest order value within the tier +- Policies with no order value are processed last in the tier, but before the implicit deny +- When multiple policies without an order value are present in a tier, they are processed in alphabetical order. However, we do not recommended relying on alphabetical ordering because it hard to operationalize. + +### How policy action rules affect traffic processing + +It is also important to understand that $[prodname] policy action rules affect how traffic and connections are processed. Let's go back to the first drop-down on the rules panel. + +Action defines what should happen when a connection matches this rule. + + + +- **Allow or Deny** - traffic is allowed or denied and the connection is handled accordingly. No further rules are processed. +- **Pass** - skips to the next tier that contains a policy that applies to the endpoint, and processes the connection. If the tier applies to the endpoint but no action is taken on the connection, the connection is dropped. +- **Log** - creates a log, and evaluation continues processing to the next rule + +## Additional resources + +The following topics go into further detail about concepts described in this tutorial: + +- [Get started with network policy](../beginners/calico-network-policy.mdx) +- [Service account selectors](../beginners/policy-rules/service-accounts.mdx) +- [Get started with tiered network policy](tiered-policy.mdx) +- [Get started with network sets](../networksets.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/policy-tiers/rbac-tiered-policies.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-tiers/rbac-tiered-policies.mdx new file mode 100644 index 0000000000..154acf01d9 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-tiers/rbac-tiered-policies.mdx @@ -0,0 +1,518 @@ +--- +description: Configure Kubernetes RBAC to control which users can edit Calico Cloud policies in each tier across connected clusters. +--- + +# Configure RBAC for tiered policies + +## Big picture + +Configure fine-grained user access controls for tiered policies. + +## Value + +Self-service is an important part of CI/CD processes for containerization and microservices. $[prodname] provides fine-grained access control (RBAC) for: + +- $[prodname] policy and tiers +- Kubernetes network policy + +## Concepts + +### Standard Kubernetes RBAC + +$[prodname] implements the standard **Kubernetes RBAC Authorization APIs** with `Role` and `ClusterRole` types. The $[prodname] API server integrates with Kubernetes RBAC Authorization APIs as an extension API server. + +### RBAC for policies and tiers + +In $[prodname], global network policy and network policy resources are associated with a specific tier. Admins can configure access control for these $[prodname] policies using standard Kubernetes `Role` and `ClusterRole` resource types. This makes it easy to manage RBAC for both Kubernetes network policies and $[prodname] tiered network policies. RBAC permissions include managing resources using the $[prodname] web console, and `kubectl`. + +### Fine-grained RBAC for policies and tiers + +RBAC permissions can be split by resources ($[prodname] and Kubernetes), and by actions (CRUD). Tiers should be created by administrators. Full CRUD operations on tiers is synonymous with full management of network policy. Full management to network policy and global network policy also requires `GET` permissions to 1) any tier a user can view/manage, and 2) the required access to the tiered policy resources. + +Here are a few examples of how you can fine-tune RBAC for tiers and policies. + +| **User** | **Permissions** | +| --------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| Admin | The default **tigera-network-admin** role lets you create, update, delete, get, watch, and list all $[prodname] resources (full control). Examples of limiting Admin access:
  • List tiers only
  • List only specific tiers
| +| Non-Admin | The default **tigera-ui-user** role allows users to only list $[prodname] policy and tier resources. Examples of limiting user access:
  • Read-only access to all policy resources across all tiers, but only write access for NetworkPolicies with a specific tier and namespace.
  • Perform any operations on NetworkPolicies and GlobalNetworkPolicies.
  • List tiers only.
  • List or modify any policies in any tier. Fully manage only Kubernetes network policies in the **default** tier, in the **default** namespace, with read-only access for all other tiers.
| + +### RBAC definitions for $[prodname] network policy + +To specify per-tier RBAC for the $[prodname] network policy and $[prodname] global network policy, use pseudo resource kinds and names in the `Role` and `ClusterRole` definitions. For example, + +```yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: tier-default-reader +rules: + - apiGroups: ['projectcalico.org'] + resources: ['tiers'] + resourceNames: ['default'] + verbs: ['get'] + - apiGroups: ['projectcalico.org'] + resources: ['tier.networkpolicies', 'networkpolicies] + resourceNames: ['default.*'] + verbs: ['get', 'list'] +``` + +Where: + +- **resources**: both `tier.globalnetworkpolicies` and `globalnetworkpolicies` for global network policies, and both `tier.networkpolicies` and `networkpolicies` for network policies. +- **resourceNames**: + - Blank - any policy of the specified kind across all tiers. + - `.*` - any policy of the specified kind within the named tier. + - `` - the specific policy of the specified kind. Because the policy name is prefixed with the tier name, this also specifies the tier. + +## Before you begin... + +**Required** + +A **cluster-admin** role with full permissions to create and modify resources. + +**Recommended** + +A rough idea of your tiered policy workflow, and who should access what. See [Configure tiered policies](tiered-policy.mdx). + +## How to + +- [Create Admin users, full permissions](#create-admin-users-full-permissions) +- [Create minimum permissions for all non-Admin users](#create-minimum-permissions-for-all-non-admin-users) + +:::note + +` kubectl auth can-i` cannot be used to check RBAC for tiered policy. + +::: + +### Create Admin users, full permissions + +Create an Admin user with full access to the $[prodname] web console (as well as everything else in the cluster) using the following command. See the Kubernetes documentation to identify users based on your chosen [authentication method](https://kubernetes.io/docs/reference/access-authn-authz/authentication/), and how to use the [RBAC resources](https://kubernetes.io/docs/reference/access-authn-authz/rbac/). + +``` +kubectl create clusterrolebinding permissive-binding \ + --clusterrole=cluster-admin \ + --user= +``` + +### Create minimum permissions for all non-Admin users + +All users using the $[prodname] web console should be able to create authorizationreviews and authorizationrequests as well as access +license information through the services/proxy https:calico-api:8080. + +1. Download the [min-ui-user-rbac.yaml manifest]($[tutorialFilesURL]/min-ui-user-rbac.yaml). + The roles and bindings in this file provide a minimum starting point for setting up RBAC for your users according to your specific security requirements. + This manifest provides basic RBAC to view some statistical data in the UI but does not provide permissions to + view or modify any network policy related configuration. + +1. Run the following command to replace `` with the name or email of the user you are providing permissions to: + + ```bash + sed -i -e 's///g' min-ui-user-rbac.yaml + ``` + +1. Use the following command to install the bindings: + + ```bash + kubectl apply -f min-ui-user-rbac.yaml + ``` + +## Tutorial + +This tutorial shows how to use RBAC to control access to resources and CRUD actions for a non-Admin user, John, with the username **john**. + +The RBAC examples shown will include: + +- [User cannot read policies in any tier](#user-cannot-read-policies-in-any-tier) +- [User can view all policies, and modify policies in the default namespace and tier](#user-can-view-all-policies-and-modify-policies-in-the-default-namespace-and-tier) +- [User can read policies only in both the default tier and namespace](#user-can-read-policies-only-in-both-the-default-tier-and-namespace) +- [User can read policies only in both a specific tier and in the default namespace](#user-can-read-policies-only-in-both-a-specific-tier-and-in-the-default-namespace) +- [User can only view a specific tier](#user-can-only-view-a-specific-tier) +- [User can read all policies across all tiers and namespaces](#user-can-read-all-policies-across-all-tiers-and-namespaces) +- [User has full control over policies only in both a specific tier and in the default namespace](#user-has-full-control-over-policies-only-in-both-a-specific-tier-and-in-the-default-namespace) + +### User cannot read policies in any tier + +User 'john' is forbidden from reading policies in any tier (**default** tier, and **net-sec** tier). + +When John issues the following command: + +```bash +kubectl get networkpolicies.p +``` + +It returns: + +``` +Error from server (Forbidden): networkpolicies.projectcalico.org is forbidden: User "john" cannot list networkpolicies.projectcalico.org in tier "default" and namespace "default" (user cannot get tier) +``` + +Similarly, when John issues this command: + +```bash +kubectl get networkpolicies.p -l projectcalico.org/tier==net-sec +``` + +It returns: + +``` +Error from server (Forbidden): networkpolicies.projectcalico.org is forbidden: User "john" cannot list networkpolicies.projectcalico.org in tier "net-sec" and namespace "default" (user cannot get tier) +``` + +:::note + +The .p' extension (`networkpolicies.p`) is short +for "networkpolicies.projectcalico.org" and used to +differentiate it from the Kubernetes NetworkPolicy resource and +the underlying CRDs (if using the Kubernetes Datastore Driver). + +::: + +:::note + +The label for selecting a tier is `projectcalico.org/tier`. +When a label selector is not specified, the server defaults the selection to the +`default` tier. Alternatively, a field selector (`spec.tier`) may be used to select +a tier. + +```bash +kubectl get networkpolicies.p --field-selector spec.tier=net-sec +``` + +::: + +### User can view all policies, and modify policies in the default namespace and tier + +1. Download the [`read-all-crud-default-rbac.yaml` manifest]($[tutorialFilesURL]/read-all-crud-default-rbac.yaml). + +1. Run the following command to replace `` with the `name or email` of + the user you are providing permissions to: + + ```bash + sed -i -e 's///g' read-all-crud-default-rbac.yaml + ``` + +1. Use the following command to install the bindings: + + ```bash + kubectl apply -f read-all-crud-default-rbac.yaml + ``` + +The roles and bindings in this file provide the permissions to read all policies across all tiers and to fully manage +policies in the **default** tier and **default** namespace. This file includes the minimum required `ClusterRole` and `ClusterRoleBinding` definitions for all UI users (see `min-ui-user-rbac.yaml` above). + +### User can read policies only in both the default tier and namespace + +In this example, we give user 'john' permission to read policies only in both the **default** tier and namespace. + +```yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: tigera-example-get-default-tier +rules: + # To access Calico policy in a tier, the user requires "get" access to that tier. +- apiGroups: ["projectcalico.org"] + resources: ["tiers"] + resourceNames: ["default"] + verbs: ["get"] + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: tigera-example-read-policies-in-default-tier +rules: + # This allows "get" and "list" of the Calico NetworkPolicy resources in the default tier. +- apiGroups: ["projectcalico.org"] + resources: ["tier.networkpolicies"] + resourceNames: ["default.*"] + verbs: ["get", "list"] + +--- + +# tigera-example-get-default-tier is applied globally +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: john-can-get-default-tier +subjects: +- kind: User + name: john + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: tigera-example-get-default-tier + apiGroup: rbac.authorization.k8s.io + +--- + +# tigera-example-read-policies-in-default-tier is applied per-namespace +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: john-can-read-policies-in-default-tier-and-namespace +subjects: +- kind: User + name: john + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: tigera-example-read-policies-in-default-tier + apiGroup: rbac.authorization.k8s.io +``` + +With the above, user john is able to list all NetworkPolicy resources in the **default** tier: + +```bash +kubectl get networkpolicies.p --all-namespaces +``` + +With some example policies on the cluster, returns: + +``` +NAMESPACE NAME CREATED AT +blue default.calico-np-blue-ns-default-tier 2021-07-26T09:05:11Z +default default.calico-np-default-ns-default-tier 2021-07-26T09:05:11Z +green default.calico-np-green-ns-default-tier 2021-07-26T09:05:13Z +red default.calico-np-red-ns-default-tier 2021-07-26T09:05:12Z +yellow default.calico-np-yellow-ns-default-tier 2021-07-26T09:05:13Z +``` + +As intended, user john can only examine those in the **default** namespace: + +```bash +kubectl get networkpolicies.p default.calico-np-green-ns-default-tier -o yaml -n=green +``` + +Correctly returns: + +``` +Error from server (Forbidden): networkpolicies.projectcalico.org "default.calico-np-green-ns-default-tier" is forbidden: User "john" cannot get networkpolicies.projectcalico.org in tier "default" and namespace "green" +``` + +John also still cannot access tier **net-sec**, as intended: + +```bash +kubectl get networkpolicies.p -l projectcalico.org/tier==net-sec +``` + +This returns: + +``` +Error from server (Forbidden): networkpolicies.projectcalico.org is forbidden: User "john" cannot list networkpolicies.projectcalico.org in tier "net-sec" and namespace "default" (user cannot get tier) +``` + +### User can read policies only in both a specific tier and in the default namespace + +Let's assume that the kubernetes-admin gives user 'john' the permission to list the policies in tier **net-sec**, but only examine the detail of the policies that are also in the **default** namespace. +To provide these permissions to user 'john', use the following `ClusterRoles`,`ClusterRoleBinding` and `RoleBinding`. + +```yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: tigera-example-get-net-sec-tier +rules: + # To access Calico policy in a tier, the user requires "get" access to that tier. +- apiGroups: ["projectcalico.org"] + resources: ["tiers"] + resourceNames: ["net-sec"] + verbs: ["get"] + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: tigera-example-read-policies-in-net-sec-tier +rules: + # This allows "get" and "list" of the Calico NetworkPolicy resources in the net-sec tier. +- apiGroups: ["projectcalico.org"] + resources: ["tier.networkpolicies"] + resourceNames: ["net-sec.*"] + verbs: ["get", "list"] + +--- + +# tigera-example-get-net-sec-tier is applied globally +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: john-can-get-net-sec-tier +subjects: +- kind: User + name: john + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: tigera-example-get-net-sec-tier + apiGroup: rbac.authorization.k8s.io + +--- + +# tigera-example-read-policies-in-net-sec-tier is applied per-namespace +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: john-can-read-policies-in-net-sec-tier-and-namespace +subjects: +- kind: User + name: john + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: tigera-example-read-policies-in-net-sec-tier + apiGroup: rbac.authorization.k8s.io +``` + +### User can only view a specific tier + +In this example, the following `ClusterRole` and `ClusterRoleBinding` can be used to provide 'get' access to the **net-sec** +tier. This has the effect of making the **net-sec** tier visible in the $[prodname] web console (including listing the names of the policies it contains). + +However, to modify or view the details of policies within the **net-sec** tier, additional RBAC permissions would be required. + +```yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: tigera-example-make-net-sec-tier-visible +rules: + # To access Calico policy in a tier, the user requires "get" access to that tier. +- apiGroups: ["projectcalico.org"] + resources: ["tiers"] + resourceNames: ["net-sec"] + verbs: ["get"] + +--- + +# tigera-example-make-net-sec-tier-visible is applied globally +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: john-can-view-the-net-sec-tier +subjects: +- kind: User + name: john + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: tigera-example-make-net-sec-tier-visible + apiGroup: rbac.authorization.k8s.io +``` + +### User can read all policies across all tiers and namespaces + +In this example, a single `ClusterRole` is used to provide read access to all policy resource types across all tiers. In this case, there is no need to use both `ClusterRoleBindings` and `RoleBindings` to map these abilities to the target user, because the intention is to for the policy to apply to all current and future namespaces on the cluster, so a `ClusterRoleBinding` provides the desired granularity. + +```yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: tigera-example-all-tiers-and-namespaces-policy-reader +rules: + # To access Calico policy in a tier, the user requires "get" access to that tier. + # Not specifying any specific "resourceNames" provides access to all tiers. +- apiGroups: ["projectcalico.org"] + resources: ["tiers"] + verbs: ["get"] + # This allows read access to the Kubernetes NetworkPolicy resources (these are always in the default tier). +- apiGroups: ["networking.k8s.io", "extensions"] + resources: ["networkpolicies"] + verbs: ["get","watch","list"] + # This allows read access to the Calico NetworkPolicy and GlobalNetworkPolicies. + # Not specifying any specific "resourceNames" provides access to them in all tiers. +- apiGroups: ["projectcalico.org"] + resources: ["tier.networkpolicies","tier.globalnetworkpolicies"] + verbs: ["get","watch","list"] + +--- + +# tigera-example-all-tiers-and-namespaces-policy-reader is applied globally, with a single ClusterRoleBinding, +# since all the rules it contains apply to all current and future namespaces on the cluster. +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: read-all-tier +subjects: +- kind: User + name: john + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: tigera-example-all-tiers-and-namespaces-policy-reader + apiGroup: rbac.authorization.k8s.io +``` + +### User has full control over policies only in both a specific tier and in the default namespace + +In this example, two `ClusterRole` objects are used to provide full access control of Calico NetworkPolicy +resource types in the **net-sec** tier: + +- The `tiers` resource is bound to a user using a `ClusterRoleBinding`, because it is a global resource. + This results in the user having the ability to read the contents of the tier across all namespaces. +- The `networkpolicies` resources are bound to a user using a `RoleBinding`, because the aim in this + case was to make them CRUD-able only in the default namespace. + You only need this one `ClusterRole` to be defined, but it can be applied to different namespaces + using additional `RoleBinding` objects. If the intention was to apply it to all current and future namespaces, + a `ClusterRoleBinding` could be used. + +```yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: tigera-example-get-net-sec-tier +rules: + # To access Calico policy in a tier, the user requires "get" access to that tier. +- apiGroups: ["projectcalico.org"] + resources: ["tiers"] + resourceNames: ["net-sec"] + verbs: ["get"] + +--- + +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: tigera-example-crud-policies-in-net-sec-tier +rules: + # This allows full CRUD access to the Calico NetworkPolicy resources in the net-sec tier. +- apiGroups: ["projectcalico.org"] + resources: ["tier.networkpolicies"] + resourceNames: ["net-sec.*"] + verbs: ["*"] + +--- + +# tigera-example-get-net-sec-tier is applied globally +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: john-can-get-net-sec-tier +subjects: +- kind: User + name: john + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: tigera-example-get-net-sec-tier + apiGroup: rbac.authorization.k8s.io + +--- + +# tigera-example-crud-policies-in-net-sec-tier is applied per-namespace +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: john-can-crud-policies-in-net-sec-tier-and-namespace +subjects: +- kind: User + name: john + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: tigera-example-crud-policies-in-net-sec-tier + apiGroup: rbac.authorization.k8s.io +``` diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/policy-tiers/tiered-policy.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-tiers/tiered-policy.mdx new file mode 100644 index 0000000000..7a77b79c9a --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-tiers/tiered-policy.mdx @@ -0,0 +1,242 @@ +--- +description: How tiered policy works in Calico Cloud — evaluation order, pass actions, and using tiers to enforce microsegmentation across connected clusters. +--- + +# Get started with policy tiers + +## Seamless network policy integration + +**Network policy** is the primary tool for securing a Kubernetes network. It lets you restrict network traffic in your cluster so only the traffic that you want to flow is allowed. $[prodname] provides more robust policy than Kubernetes, but you can use them together -- seamlessly. $[prodname] supports: + +- $[prodname] network policy, (namespaced) +- $[prodname] global network policy (non-namespaced, global) +- Kubernetes network policy + +## Tiers: what and why? + +**Tiers** are a hierarchical construct used to group policies and enforce higher precedence policies that cannot be circumvented by other teams. As you will learn in this tutorial, tiers have built-in features that support workload microsegmentation. + +All $[prodname] and Kubernetes network policies reside in tiers. You can start "thinking in tiers" by grouping your teams and types of policies within each group. For example, we recommend these three tiers (platform, security, and application). + +![policy-types](/img/calico-enterprise/policy-types.png) + +Next, you can determine the priority of policies in tiers (from top to bottom). In the following example, the platform and security tiers use $[prodname] global network policies that apply to all pods, while developer teams can safely manage pods within namespaces using Kubernetes network policy for their applications and microservices. + +![policy-tiers](/img/calico-enterprise/policy-tiers.png) + +## Create a tier and policy + +To create a tier and policy in the web console: + +1. In the left navbar, click **Policies**. +1. On the **Policies Board**, click **Add Tier**. +1. Name the tier, select **Order, Add after** `tigera-security`, and save. +1. To create a policy in the tier, click **+ Add policy**. + +You can export all policies or a single policy to a YAML file. + +Here is a sample YAML that creates a security tier and uses `kubectl` to apply it. + +```yaml +apiVersion: projectcalico.org/v3 +kind: Tier +metadata: + name: security +spec: + order: 300 +``` + +```bash +kubectl apply -f security.yaml +``` + +## The default tier: + +The default tier is created during installation and has the order of 1,000,000. This value is fixed, and cannot be changed. The desire is to evaluate policies in the default tier as last. + +![default-tier](/img/calico-enterprise/default-tier.png) + +The default tier is where: + +- You manage all Kubernetes network policies +- Network and global network policies are placed when you upgrade from Project Calico to $[prodname] +- Recommended policies are placed when you use the **Recommend a policy** feature + +## The admin network policy tier: + +The `Tier` with name `adminnetworkpolicy` is where all [Kubernetes admin network policies](https://network-policy-api.sigs.k8s.io/reference/examples/) reside. It is automatically created +during installation and has the order of 1,000 and a default action of `Pass`. This is fixed, and should not be changed. + +In addition to `AdminNetworkPolicy` resources, you can also add $[prodname] `NetworkPolicy`/`GlobalNetworkPolicy` resources +to the admin network policy `Tier`. The `AdminNetworkPolicy` `priority` field maps to the `order` field in the $[prodname] +`NetworkPolicy`/`GlobalNetworkPolicy` resources so that the two types of policy can be ordered together. + +## The baseline admin network policy tier: + +The `Tier` with name `baselineadminnetworkpolicy` is where the singleton [Kubernetes baseline admin network policy](https://network-policy-api.sigs.k8s.io/reference/examples/) resides. It is automatically created +during installation and has the order of 10,000,000 and a default action of `Pass`. This is fixed, and should not be changed. + +In addition to the `BaselineAdminNetworkPolicy` resource, you can also add $[prodname] `NetworkPolicy`/`GlobalNetworkPolicy` resources +to the baseline admin network policy `Tier`. The `BaselineAdminNetworkPolicy` is treated as if it had an `order` of 1,000, allowing $[prodname] +`NetworkPolicy`/`GlobalNetworkPolicy` resources to be inserted with higher/lower precedence than the `BaselineAdminNetworkPolicy`. + + +## System tiers + +System tiers are added during installation and are hidden by default. + +- **calico-system** tier contains policies to secure $[prodname] components and are controlled by the Tigera Operator. These policies should not be edited, and the tier should not be moved. Inadvertent changes are automatically reverted by the Operator to ensure your cluster is always protected. + +:::warning + +Although it is possible to change the behavior of the `calico-system` tier using adjacent tiers, it is not a trivial task. You can break critical cluster traffic and impact the operation of $[prodname]. To prevent loss of cluster services, see [Change calico-system tier behavior](calico-system.mdx), and contact Support for help. + +::: + +## Moving tiers + +You can move tiers by dragging and moving them in the graphical sequence, but all tiers must be visible first before you reorder tiers. + +To show all tiers, click **View** and select all of the tiers in the Show tiers list. + +![hidden-tiers](/img/calico-enterprise/moving-tiers.png) + +Now you can reorder tiers by dragging and moving them. + +## Tier order + +Tiers are sorted by their orders (from left to right), starting from the lowest order (the highest priority) tiers. + +![tier-order](/img/calico-enterprise/tier-order.png) + +In the example above, tier priorities are as follows: + +- **security tier** - is higher priority than platform tier +- **platform tier** - is higher priority than default tier +- **default tier** - is the lowest priority + +The tier you put as the highest priority (after system tiers), depends on your environment. In compliance-driven environments, the security tier may be the highest priority (as shown above). There is no one-size-fits-all order. + +## Policy processing + +Policies are processed in sequential order from lowest order (highest priority) to highest order (lowest priority), which is from top to bottom in the following diagram. + +![policy-processing](/img/calico-enterprise/policy-processing.png) + +Two mechanisms drive how traffic is processed across tiered policies: + +- Labels and selectors +- Policy action rules + +It is important to understand the roles they play. + +### Labels and selectors + +Instead of IP addresses and IP ranges, network policies in Kubernetes depend on labels and selectors to determine which workloads can talk to each other. Workload identity is the same for Kubernetes and $[prodname] network policies: as pods dynamically come and go, network policy is enforced based on the labels and selectors that you define. + +The following diagrams show the relationship between all of the elements that affect traffic flow: + +- **Tiers** group and order policies +- **Policy action rules** define how to process traffic in and across tiers, and policy labels and selectors specify how groups of pods are allowed to communicate with each other and other network endpoints +- The **CNI**, **$[prodname] components**, and underlying **data plane** (iptables/eBPF) all make use of labels and selectors as part of routing traffic. + +![tier-funnel](/img/calico-enterprise/tier-funnel.png) + +### Policy action rules + +$[prodname] network policy uses action rules to specify how to process traffic/packets: + +- **Allow or Deny** - traffic is allowed or denied and the packet is handled accordingly. No further rules are processed. +- **Pass** - skips to the next tier that contains a policy that applies to the endpoint, and processes the packet. If the tier applies to the endpoint but no action is taken on the packet, the packet is dropped. +- **Log** - creates a log, and evaluation continues processing to the next rule + +### Implicit default deny + +As shown in the following diagram, at the end of each tier is an implicit default deny. This is a safeguard that helps mitigate against unsecured policy. +Because of this safeguard, you must either explicitly update tier's default action to **Pass** or apply a Pass action rule when you want traffic evaluation to continue. +In the following example, the Pass action in a policy ensures that traffic evaluation continues, and overrides the implicit default deny. + +![implicit-deny](/img/calico-enterprise/implicit-deny.svg) + +Let’s look at a Dev/Ops global network policy in a high precedence tier (Platform). The policy denies ingress and egress traffic to workloads that match selector, `env == "stage"`. To ensure that policies continue to evaluate traffic after this policy, the policy adds a Pass action for both ingress and egress. + +**Pass action rule example** + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: devops.stage-env +spec: + tier: devops + order: 255 + selector: env == "stage" + ingress: + - action: Deny + source: + selector: env != "stage" + - action: Pass + egress: + - action: Deny + destination: + selector: env != "stage" + - action: Pass + types: + - Ingress + - Egress +``` + +The other option, is to update tier's default action to **Pass** to override the implicit default deny. + +**Pass default action tier example** + +```yaml +apiVersion: projectcalico.org/v3 +kind: Tier +metadata: + name: devops +spec: + order: 300 + defaultAction: Pass +``` + +### Policy endpoint matching across tiers + +Whoever is responsible for tier creation, also needs to understand how policy selects matching endpoints across tiers. For normal policy processing (without apply-on-forward, pre-DNAT, and do-not-track), if no policies within a tier apply to endpoints, the tier is skipped, and the tier's implicit deny behavior is not executed. + +In the following example, **policy D** in the Security tier includes a Pass action rule because we want traffic evaluation to continue to the next tier in sequence. In the Platform tier, there are no selectors in policies that match endpoints so the tier is skipped, including the end of tier deny. Evaluation continues to the Application tier. **Policy J** is the first policy with a matching endpoint. + +![endpoint-match](/img/calico-enterprise/endpoint-match.svg) + +### Default endpoint behavior + +Also, tier managers need to understand the default behavior for endpoints based on whether the endpoint is known or unknown, and the endpoint type. As shown in the following table: + +- **Known endpoints** - $[prodname] resources that are managed by Felix +- **Unknown endpoints** - interfaces/resources not recognizable as part of our data model + +| Endpoint type | Default behavior for known endpoints | Default behavior for unknown endpoints (outside of our data model) | +| ---------------------- | ----------------------------------------------------------------------- | ------------------------------------------------------------------ | +| Workload, $[prodname] | Deny | Deny | +| Workload, Kubernetes | Allow ingress from same Kubernetes namespace; allow all egress | Deny | +| Host | Deny. With exception of auto host endpoints, which get `default-allow`. | Fall through and use iptables rules | + +## Best practices for tiered policy + +To control and authorize access to $[prodname] tiers, policies, and Kubernetes network policies, you use Kubernetes RBAC. Security teams can prevent unauthorized viewing or modification of higher precedence (lower order) tiers, while still allowing developers or service owners to manage the detailed policies related to their workloads. + +We recommend: + +- Limit tier creation permissions to Admin users only; creating and reordering tiers affects your policy processing workflow + +- Limit full CRUD operations on tiers and policy management to select Admin users + +- Review your policy processing whenever you add/reorder tiers + + For example, you may need to update Pass action rules to policies before or after the new tier. Intervening tiers may require changes to policies before and after, depending on the endpoints. + +- Use the **policy preview** feature to see effects of policy in action before enforcing it, and use the **staged network policy** feature to test the entire tier workflow before pushing it to production + +## Additional resources + +- For details on using RBAC for fine-grained access to tiers and policies, see [Configure RBAC for tiered policies](rbac-tiered-policies.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/policy-troubleshooting.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-troubleshooting.mdx new file mode 100644 index 0000000000..4f1bcf0bbe --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/policy-troubleshooting.mdx @@ -0,0 +1,42 @@ +--- +description: Troubleshooting guide for Calico Cloud policy implementation problems in connected clusters — denied traffic, missing rules, and tier-evaluation surprises. +--- + +# Troubleshoot policies + +### Problem + +I created ​​my first egress policy with default deny behavior, but now I’ve blocked other traffic. + +### Solution + +In Kubernetes, when there are no egress policies that apply to an endpoint, all egress traffic is allowed. However, as soon as you add the first egress policy to an endpoint, $[prodname] switches to default deny and blocks everything else; this is part of our zero trust network policy model. For new users of $[prodname], this is unexpected behavior (but it’s required by both Kubernetes and $[prodname] policy specs.) + +For egress policy in particular, you may not be used to worrying about “system-level” egress traffic that is now suddenly blocked. For example, most workloads rely on DNS, but you may not have thought of this when writing your policy. So you end up with this problem loop: you allow HTTP traffic, but then your DNS traffic gets blocked, but then HTTP traffic stops working because it relies on DNS to function. + +A natural response to this issue is to add an egress rule to allow DNS(!). For example, you add an egress rule “allow UDP to port 53 to namespace kube-system”. In some systems (OpenShift), the DNS pod actually listens on port 5353, not port 53. However, the DNS Service DNATs the traffic from port 53 to port 5353, hiding that detail from the DNS client. $[prodname] then blocks the traffic because it sees the traffic after the DNAT. So $[prodname] sees port 5353, not the expected port 53. + +The solution is to define policy for workload services, not for ports used by workloads. For help, see [Policy for services](beginners/services/index.mdx). + +### Problem + +Traffic is blocked, even though I allow it in a policy. + +### Solution + +The problem of blocking traffic can reside in your tier, or a different tier. + +1. **Check policies in your tier** + + Go to your policy and see if there is a higher precedent policy in the tier that is blocking processing. + + - If that is not the problem, go to step 2. + - If that is the problem, and if it makes sense for the traffic, you can reorder the policies in the tier. If you cannot, you must change the policy that is dropping traffic to allow your traffic flow using a Pass action rule. + +2. **Check policies in other tiers** + + Go to the next applicable higher precedent tier for your workload to see if a policy in that tier is blocking traffic. The policy at the end of the tier could be blocking traffic because the default behavior at the end of a tier is to drop traffic as part of zero trust. To unblock traffic, add a **Pass action rule** to the policy, or create a **Pass policy**. + +For help with visibility, use Service Graph to see how traffic is passed. Click on your flow, and view details in the right panel. + +For help with Pass action rules, see [Get started with tiered policy](policy-tiers/tiered-policy.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/recommendations/index.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/recommendations/index.mdx new file mode 100644 index 0000000000..32ee67a4a6 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/recommendations/index.mdx @@ -0,0 +1,11 @@ +--- +description: Use Calico Cloud policy recommendations to generate baseline network policy for unprotected namespaces from observed flow logs in connected clusters. +hide_table_of_contents: true +--- + +# Policy recommendations + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/recommendations/learn-about-policy-recommendations.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/recommendations/learn-about-policy-recommendations.mdx new file mode 100644 index 0000000000..0bbc217bda --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/recommendations/learn-about-policy-recommendations.mdx @@ -0,0 +1,136 @@ +--- +description: Tutorial walkthrough of the Calico Cloud policy recommendations engine — what it generates, how to review it, and how to promote it to enforced. +--- + +# Policy recommendations tutorial + +## Big picture + +In this tutorial, we show you how recommendations are generated using flow logs in your cluster for traffic to/from namespaces, network sets, private network IPs and public domains. + +### Create resources for the tutorial + +Because the policy recommendation feature requires traffic between endpoints, this step provides these resources for this hands-on tutorial. If your cluster is already generating traffic for policy recommendations, you can skip this step and follow along using your own cluster. + +1. Configure felix for fast flow logs collection + + ```bash + kubectl patch felixconfiguration.p default -p '{"spec":{"flowLogsFlushInterval":"10s"}}' + ``` + +1. Download the [policy recommendation tutorial deployment]($[tutorialFilesURL]/policy-recommendation-deployments.yaml) YAML. + +1. Use the following command to create the necessary resources: + + ```bash + kubectl apply -f policy-recommendation-deployments.yaml + ``` + +### Enable policy recommendation + +1. In the web console left navbar, click the **Policies** icon. +1. Select **Recommendations**. +1. Click on **Enable Policy Recommendations**. + +Wait for the recommendations to be generated. Unless otherwise configured, recommendations will take at least 2m30s to be generated, which is default time for the [Processing Interval](../../reference/resources/policyrecommendations.mdx#spec) setting. + +Once ready, the recommendations will be listed in the main page, under the **Recommendations** tab. + +### Understand the policy recommendation + +You should find a recommendation named `curl-ns` (appended with a five character suffix, like `-vfzgh`) with policy selector: +``` +Policy Label selector: [[projectcalico.org/namespace == 'curl-ns']] +``` +meaning that this policy pertains to the traffic originating from or destined for the `curl-ns` namespace. + +The policy will display a list of ingress rules: +``` +Allow:Protocol is TCP +From: Namespaces [[projectcalico.org/name == 'service-ns']] +To:Ports [Port is 80 ] +``` +allows ingress traffic, for protocol TCP, on port 80, from the `service-ns` namespace. + +A list of egress rules: +``` +Allow:Protocol is TCP +To:Ports [Port is 8080 ] Domains [www.tigera.io] +``` +allows egress traffic, for protocol TCP, on port 8080, to domain `www.tigera.io`. + +``` +Allow:Protocol is TCP +To:Ports [Port is 80 ] Namespaces [[projectcalico.org/name == 'service-ns']] +``` +allows egress traffic, for protocol TCP, on port 80, to the `service-ns` namespace. + +``` +Allow:Protocol is UDP +To:Ports [Port is 53 ] Namespaces [[projectcalico.org/name == 'kube-system']] +``` +allows egress traffic, for protocol UDP, on port 53, to the `kube-system` namespace. + +``` +Allow:Protocol is TCP +To:Ports [Port is 80 ] Endpoints [[projectcalico.org/name == 'public-ips' and projectcalico.org/kind == 'NetworkSet']] Namespaces global() +``` +allows egress traffic, for protocol TCP, on port 80, to IPs defined in the global network set named: `public-ips`. + +``` +Allow:Protocol is TCP +To:Ports [Port is 8080 ] Nets [Is 10.0.0.0/8 OR Is 172.16.0.0/12 OR Is 192.168.0.0/16 ] +``` +allows egress traffic, for protocol TCP, on port 8080, to private range IPs. + +``` +Allow:Protocol is TCP +To:Ports [Port is 80 ] +``` +allows egress traffic, for protocol TCP, on port 80, to public range IPs. + +### Investigate the flows that are used to generate the policy rules + +To view flow logs in Service Graph: + +1. In the web console left navbar, click **Service Graph**. +1. Select **Default** under the VIEWS option. +1. In the bottom pane you will see flow logs in the Flows tab. + +To generate rules, the recommendation engine queries for flow logs that are not addressed by any other policy in the cluster. Subsequently, it builds the missing policies necessary for allowing that traffic. + +### Understand the flow logs used in policy recommendations + +To get a better understanding of which flows contributed to generating the rules in your policy, select **Filter Flows** + +* To find the flows that were used to generate the egress to global network set rule, add: +``` +source_namespace = "curl-ns" AND dest_name_aggr = "public-ips" +``` + +* To find the flows that generated the egress rule to namespace `kube-system`, define query: +``` +source_namespace = "curl-ns" AND dest_namespace = "kube-system" +``` + +You'll notice that each of the flow logs contains a field named, `policies` with a entry like: +``` +1|__PROFILE__|__PROFILE__.kns.curl-ns|allow|0 +``` +meaning that the particular flow was not addressed by any other policy within your cluster. + +You will also find input like: +``` +0|namespace-isolation|curl-ns/namespace-isolation.staged:curl-ns-vfzgh|allow|3 +``` +indicating that the 3rd rule defined in policy **curl-ns-vfzgh**, will allow traffic defined by this flow, once the policy is enforced. + +### Examine policy traffic + +Examine the **Allowed Bytes** field in the **Recommendations** tab for the `curl-ns-recommendation` policy to get a sense of the total bytes allowed by the policy. + +Examine the **Allowed/sec** of each rule in the policy to get a sense of the quantity of traffic allowed per second by the rule in question. + +### When policy recommendations are not generated + +You may wonder why you are not getting policy recommendations, even though there is traffic between endpoints. This is because policy recommendations are generated only for flows that are not captured by any other policy in your cluster. To see if policy is already enforcing the traffic in question, search for the flow log in question, examine the `policies` field, and verify that no other enforced policy allows or denies traffic for that flow. diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/recommendations/policy-recommendations.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/recommendations/policy-recommendations.mdx new file mode 100644 index 0000000000..aa907a5543 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/recommendations/policy-recommendations.mdx @@ -0,0 +1,186 @@ +--- +description: Run continuous Calico Cloud policy recommendations so unprotected namespaces and workloads pick up baseline policy automatically. +--- + +# Enable policy recommendations + +## Big picture + +Use policy recommendations to automatically isolate namespaces with network policy. + +## Value + +One of the best practices for improving the security posture of your Kubernetes cluster is to implement namespace isolation with network policy. Namespace isolation helps to implement a zero-trust and least-privileged security model, where only required communication between namespaces is authorized and everything else is blocked. This helps mitigate the risk of lateral movement within a cluster in the event of an attack. + +$[prodname] makes it easy for platform operators to implement namespace isolation without experience in authoring network policy or detailed knowledge of how application workloads are communicating. $[prodname] analyzes the flow logs that are generated from workloads, and automatically recommends and stages policies for each namespace that can be used for isolation. + +## Before you begin + +**Unsupported** +- Windows nodes + +**Required RBAC** + +To enable/disable and use policy recommendations, you must have the **tigera-network-admin** role or permissions to **update**, **patch**, **get**, **list**, **watch** `projectcalico.org` resources: +* tiers +* policyrecommendationscopes +* stagednetworkpolicies +* tier.stagednetworkpolicies +* networkpolicies +* tier.networkpolicies +* globalnetworksets + +Specifically, you will need access to the `namespace-isolation` tier and to staged and network policies in the `namespace-isolation` tier. + +**Recommended** + +Basic knowledge of policies in the web console and tiers: +- [Get started with tiered network policy](../../network-policy/policy-tiers/tiered-policy) +- [Network policy tutorial](../../network-policy/policy-tiers/policy-tutorial-ui) + +**Limitations** + +Creating and managing policy recommendations is available only in the web console. + +## How to + +- Enable policy recommendations +- [Activate and review policy recommendations](#activate-and-review-policy-recommendations) +- [Review global settings for workloads](#review-global-settings-for-workloads) +- [Update policy recommendations](#update-policy-recommendations) +- [Private network recommendations](#private-network-recommendations) +- [Troubleshoot policy recommendations](#troubleshoot-policy-recommendations) +- [Disable the policy recommendations feature](#disable-policy-recommendations) + +### Enable policy recommendations **using the web console** + +1. In the left navbar in the web console, click **Policies**, **Recommendations**. +1. On the opt-in page, click **Enable Policy Recommendations**. + +The **Policy Recommendations** board is automatically displayed. + +![Policy-recommendations-board](/img/calico-cloud/policy-recommendations-board.png) + +**Notes**: + +- A policy recommendation is generated for every namespace in your cluster (unless namespaces are filtered out by an Admin using the [selector](../../reference/resources/policyrecommendations.mdx#namespacespec) in the PolicyRecommendationScope resource). +- Flow logs are continuously monitored for policy recommendations. +- Recommended policies are continuously updated until you **Add to policy board** or **Dismiss policy** using the Actions menu. +- Policy recommendations are created as **staged network policies** so you can safely observe the traffic before enforcing them. +- Traffic originating from the recommended policy's namespace is used to generate egress rules, and traffic destined for the namespace is used to define ingress rules. +- To stop policy recommendations from being processed and updated for a namespace, click the **Action** menu, **Dismiss policy**. + +### Enable policy recommendations **using kubectl**. + +To enable the policy recommendations feature, set the **RecStatus** parameter to `Enabled`, in the [Policy recommendations resource](../../reference/resources/policyrecommendations.mdx). + +```bash +kubectl patch PolicyRecommendationScope default --type='json' -p='[{"op": "replace", "path": "/spec/namespaceSpec/recStatus", "value": "Enabled"}]' +``` + +### Activate and review policy recommendations + +Policy recommendations are not enabled until you activate them and move them to the **Active** board. + +From the Policy Recommendation board, select a policy recommendation (or bulk select) and select, **Add to policy board**. Click on the **Active tab**. + +You can now view the activated policies in the **Policies Board**. In the left navbar, click **Policies**. + +Policy recommendations are added to the **namespace-isolation** tier. Note the following: + +- Staged network policy recommendations work like any other staged network policy. +- You cannot move recommended staged policies in the `namespace-isolation` tier. +- The name of the `namespace-isolation` tier is fixed and cannot be changed + +You are now ready to observe traffic flows in Policies board to verify that the policy is authorizing traffic as expected. When a policy works as expected, you can safely enforce it. See [Stage, preview impacts, and enforce policy](network-policy/staged-network-policies.mdx) for help. + +### Review global settings for workloads + +The default global settings for capturing flows for policy recommendations are based on application workloads with *frequent communication with other namespaces in your cluster*. + +Global settings are found on the Policy Recommendations board, **Action** menu. + +![Global-settings-dialog](/img/calico-cloud/global-settings.png) + +- **Stabilization Period** is the learning time to capture flow logs so that a recommendation accurately reflects the cluster's traffic patterns. + +- **Processing Interval** is the frequency to process new flow logs and refine recommendations. + +:::tip +For application workloads with less frequent communication, the stabilization period setting may not be long enough to get accurate traffic flows, so you’ll want to increase the time. We recommend that you review your workloads immediately after you enable policy recommendations and adjust the settings accordingly. +::: + +Changes to all other policy recommendations parameters require Admin permissions and can be changed using the [Policy recommendations resource](../../reference/resources/policyrecommendations.mdx). + +### Update policy recommendations + +This section describes common changes you may want to make to policy recommendations. + +#### Relearn activated recommendations + +As new namespace and components are added to a cluster, your activated policy recommendation may need to be updated to reflect those changes. If a policy recommendation has not been enforced, you’ll need to update it to allow traffic. + +1. On the **Policies Recommendations** board, click the **Active tab**, which lists the active staged network policies. +1. Select the Actions menu associated with the policy in question, and click **Dismiss policy**. +1. Click the **Dismissed tab**, select the Actions menu, and **Reactivate** the policy. + +#### Rerun policy recommendations for an enforced policy + +To generate a new recommendation for an enforced policy, delete the network policy on the **Policy** board. + +#### Stop policy recommendation updates for a namespace + +1. On the Policy Recommendations board, click the **Recommendations** tab, which lists the recommendations. +1. Select the recommendation, click the **Actions** menu, and click **Dismiss policy**. + +To reactivate a policy recommendation for a namespace, select the dismissed staged policy, and from the Actions menu, select **Reactivate**. + +### Private network recommendations + +If any flow to a private network in your cluster is found, a private rule is automatically created that contains RFC 1918 subnets, which will allow traffic to/from those endpoints. If you need to apply a more restrictive approach, create a [GlobalNetworkSet](../../reference/resources/globalnetworkset.mdx) and update it with the desired CIDR blocks. The recommendation engine will identify flows to your private IPs and generate the appropriate NetworkSet Rule. + +**Notes**: +Exclude any CIDR ranges used by the cluster for nodes and pods. + +### Troubleshoot policy recommendations + +**Problem**: I’m not seeing policy recommendations on the Policy Recommendations board. + +**Solution/workaround**: Policy recommendations are based on historical flow logs that match a request, and are generated only for flows that have not been addressed by any other policy. As such, there are times when policy recommendations will not be generated: + +- Not enough traffic history + + If you recently installed $[prodname], you may not have enough traffic history. Workloads must run for some time (around 5 days) to get “typical network traffic” for applications. + +- Traffic is covered by existing policy + + Even if your cluster has been running for a long time with traffic, the flows may already be covered by existing policies. + +To verify why there may not be any recommendations, follow these steps: + +1. Go to **Service Graph**, **Default**. +1. Filter flow logs for your namespace. +1. Investigate the content within the `policies` field for the flow logs in question. +1. Validate that no other enforced policy already addresses the flow. + +**Problem**: Why are egress-to-domain rules being generated for a Kubernetes service? + +**Solution/workaround**: The policy recommendation controller can only read the cluster domain of the cluster it runs in. If you have managed clusters with a non-default domain (`cluster.local`), the controller will treat egress traffic as though it is to a domain. + +### Disable policy recommendations + +To disable the policy recommendations feature, set the **RecStatus** parameter to `Disabled`, in the [Policy recommendations resource](../../reference/resources/policyrecommendations.mdx). + +```bash +kubectl patch PolicyRecommendationScope default --type='json' -p='[{"op": "replace", "path": "/spec/namespaceSpec/recStatus", "value": "Disabled"}]' +``` + +:::note + +When disabled, non-active staged policies in the **Policies Recommendations** board are no longer updated. Existing activated and enforced staged network policies are not affected by disabling policy recommendations. + +::: + +## Additional resources + +- [Policy best practices](../../network-policy/policy-best-practices.mdx) \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/network-policy/staged-network-policies.mdx b/calico-cloud_versioned_docs/version-24-1/network-policy/staged-network-policies.mdx new file mode 100644 index 0000000000..0bd27af9b5 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/network-policy/staged-network-policies.mdx @@ -0,0 +1,179 @@ +--- +description: Stage and preview Calico Cloud network policies from the web console to observe traffic impact across connected clusters before enforcing. +--- + +# Stage, preview impacts, and enforce policy + +## Big picture + +Stage and preview impacts on traffic before enforcing policy. + +## Value + +$[prodname] staged network policy resources lets you test the traffic impact of the policy as if it were enforced, but without changing traffic flow. You can also preview the impacts of a staged policy on existing traffic. By verifying that correct flows are allowed and denied before enforcement, you can minimize misconfiguration and potential network disruption. + +## Concepts + +### About staged policies + +The following staged policy resources have the same structure (i.e. the resource spec has the same fields) as their “enforced” counterpart. + +- Staged global network policy +- Staged network policy +- Staged Kubernetes network policy + +### Review permissions + +The default `tigera-network-admin` cluster role has the required permissions to manage the different enforced +and staged network policies. Adjust permissions for your environment. As with $[prodname] network policy and global network policies, the RBAC for $[prodname] staged network policy and staged global network policy is tier-dependent. + +## How to + +- [Create a policy recommendation](#create-a-policy-recommendation) +- [Stage a policy](#stage-a-policy) +- [Preview policy impact](#preview-policy-impact) +- [Enforce a staged policy](#enforce-a-staged-policy) +- [Stage updates to an enforced policy](#stage-updates-to-an-enforced-policy) + +### Create a policy recommendation + +One of the first things developers need to do is secure unprotected workloads with network policy. (For example, by default, Kubernetes pods accept traffic from any source.) The **Recommend policy** feature allows developers with minimal experience writing policy to secure workloads. + +Because **Recommend policy** looks at historical flow log entries that match your request, you should run your workloads for a reasonable amount of time to get "typical network traffic" for your application. + +1. In the left navbar, click **Policies**. +1. Click **Recommend a policy**. +1. Enter time range, Namespace, Name, and click **Recommend**. +1. If relevant flow logs are found within the time range for the workload endpoint, click **Preview** to assess the impact of the recommended policy, or **Stage**. + +![recommend-policy](/img/calico-enterprise/recommend-policy.png) + +### Stage a policy + +Stage a policy to test it in a near replica of a production environment. + +1. In the left navbar, click **Policies**. +1. In a tier, click **Add Policy**. +1. Create your policy and click **Stage** to save and stage it. + + +### Preview policy impact + +Before enforcing a staged policy, it is a best practice to use the **Preview** feature to avoid unintentionally exposing or blocking other network traffic. + +1. From the **Policies Board**, select a staged policy and click **Edit policy**. +1. Make some edits and click **Preview**. + + +### Enforce a staged policy + +1. From **Policies Board**, click a staged policy. +1. Click**Edit policy**, make changes and click **Enforce**. The staged policy is deleted and the enforced policy is created/updated (depending on whether it already exists). + +### Stage updates to an enforced policy + +1. From the **Policies Board**, open an enforced policy. +1. In **View Policy**, click **Edit policy**. +1. Make your changes, and click **Preview**. Depending on the results, you can click **Stage** or **Enforce**. + +You can also use custom resources to stage Kubernetes and $[prodname] policies, and apply them using `kubectl`. Here are sample YAML files. + +**Example: StagedGlobalNetworkPolicy** + +```yaml +apiVersion: projectcalico.org/v3 +kind: StagedGlobalNetworkPolicy +metadata: + name: default.allow-tcp-6379 +spec: + tier: default + selector: role == 'database' + types: + - Ingress + - Egress + ingress: + - action: Allow + protocol: TCP + source: + selector: role == 'frontend' + destination: + ports: + - 6379 + egress: + - action: Allow +``` + +**Example: StagedNetworkPolicy** + +```yaml +apiVersion: projectcalico.org/v3 +kind: StagedNetworkPolicy +metadata: + name: default.allow-tcp-6379 + namespace: default +spec: + tier: default + selector: role == 'database' + types: + - Ingress + - Egress + ingress: + - action: Allow + protocol: TCP + source: + selector: role == 'frontend' + destination: + ports: + - 6379 + egress: + - action: Allow +``` + +**Example: StagedKubernetesNetworkPolicy** + +```yaml +apiVersion: projectcalico.org/v3 +kind: StagedKubernetesNetworkPolicy +metadata: + name: test-network-policy + namespace: default +spec: + podSelector: + matchLabels: + role: db + policyTypes: + - Ingress + - Egress + ingress: + - from: + - ipBlock: + cidr: 172.17.0.0/16 + except: + - 172.17.1.0/24 + - namespaceSelector: + matchLabels: + project: myproject + - podSelector: + matchLabels: + role: frontend + ports: + - protocol: TCP + port: 6379 + egress: + - to: + - ipBlock: + cidr: 10.0.0.0/24 + ports: + - protocol: TCP + port: 5978 +``` + +## Additional resources + +- [Staged global network policy](../reference/resources/stagedglobalnetworkpolicy.mdx) +- [Staged network policy](../reference/resources/stagednetworkpolicy.mdx) +- [Staged Kubernetes network policy](../reference/resources/stagedkubernetesnetworkpolicy.mdx) +- For details on how to configure RBAC for staged policy resources, see [Configuring RBAC for tiered policy](policy-tiers/rbac-tiered-policies.mdx) +- For details on staged policy metrics, see + - [Flow logs](../observability/elastic/flow/datatypes.mdx) + - [Prometheus metrics](../operations/monitor/metrics/index.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/add-maglev-load-balancing.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/add-maglev-load-balancing.mdx new file mode 100644 index 0000000000..01cd6ccaf3 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/add-maglev-load-balancing.mdx @@ -0,0 +1,94 @@ +--- +description: Switch a Kubernetes service to Maglev consistent-hash load balancing on the Calico Cloud eBPF data plane for stable backend selection in connected clusters. +--- + +# Add Maglev load balancing to a service + +This guide shows you how to enable Maglev load balancing for external traffic to a service. + +Maglev load balancing opens up the possibility of using $[prodname] as a distributed, horizontally +scalable load balancer for external connections, and allows for such connections to fail over from +one load balancing node to another during a node outage. + +One case where you might enable Maglev for your service is when you wish to advertise the service IP out to your broader network as an ECMP route. +$[prodname] nodes can be used as the nexthops for such a route. These nexthops allow external connections to that service to survive +nexthop downtime and churn, say, in the case where a node must be taken down for maintenance. +This is because traffic may failover onto another path when the normal path stops working. + + +## Limitations + +Note Maglev load balancing cannot be used with the following: +* Internal (pod-to-service) traffic. +* NodePorts - as NodePort connections are always tied to a single node, and cannot failover to another. +* Kubernetes External Traffic Policy. Maglev chooses from a lookup table of all valid backends when performing backend selection. + + +## Prerequisites + +* Your cluster uses the eBPF data plane with [direct server return mode](../../operations/ebpf/enabling-ebpf.mdx#try-out-direct-server-return-mode). +* All your nodes are running on Linux. +* You have a service with a VIP/External-IP, possibly allocated by $[prodname] [LB IPAM](../ipam/service-loadbalancer.mdx), which you are advertising outside of your cluster. + + +## Enable Maglev load balancing for a service + +To enable Maglev load balancing for a service, add the following annotation: + +```yaml +lb.projectcalico.org/external-traffic-strategy: "maglev" +``` + +Or annotate a pre-existing service with: + +```bash +kubectl -n annotate service 'lb.projectcalico.org/external-traffic-strategy=maglev' +``` + +Replace the following: +* ``: The namespace of the service. +* ``: The name of the service. + + +## Verify that $[prodname] has configured Maglev load balancing for the service + +After annotating the service in-question, check any $[noderunning] container for an entry +in the $[prodname] eBPF NAT table with the "maglev" flag. + +```bash title='Example NAT table dump' +kubectl -n calico-system exec -- calico-node -bpf nat dump +``` + +Replace `` with the name of any calico-node pod. + +Check the command output for an entry with your service's IP, and the "maglev" flag mentioned in the same line: + +``` +10.101.205.28 port 80 proto 6 id 7 count 4 local 1 flags maglev + 7:0 192.168.95.210:9000 + 7:1 192.168.163.6:9000 + 7:2 192.168.221.69:9000 + 7:3 192.168.28.136:9000 +``` + + +## Additional notes + +### BPF map sizing for large deployments + +In order to evenly distribute traffic, consistent-hash-based load balancing algorithms like Maglev must allocate large lookup-tables (LUTs), +leading to a larger memory footprint than basic algorithms e.g. random-selection, round-robin. For this reason, the feature is enabled on a +per-service basis, rather than globally applying to all services. + +In $[prodname], we program these LUTs to the eBPF data plane as maps which can grow up to a maximum size. +If you plan to provision more than 100 services, or if you plan for any one service to have more than 100 endpoints, then the $[prodname] +FelixConfiguration should be updated to reflect that: + - [BPFMaglevMaxEndpointsPerService](../../reference/resources/felixconfig.mdx#bpfMaglevMaxEndpointsPerService) + - [BPFMaglevMaxServices](../../reference/resources/felixconfig.mdx#bpfMaglevMaxServices) + + +## Additional resources +* [Prometheus Metrics for Maglev](../../reference/component-resources/node/felix/prometheus.mdx#load-balancer-metrics) +* [FelixConfiguration](../../reference/resources/felixconfig.mdx) +* [Enable the eBPF data plane](../../operations/ebpf/enabling-ebpf.mdx) +* [Advertise Service IPs with BGP](advertise-service-ips.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/advertise-service-ips.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/advertise-service-ips.mdx new file mode 100644 index 0000000000..221b8d79bb --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/advertise-service-ips.mdx @@ -0,0 +1,270 @@ +--- +description: Advertise Kubernetes service cluster IPs and external IPs out of a Calico Cloud connected cluster over BGP so upstream routers can reach them directly. +--- + +# Advertise Kubernetes service IP addresses + +## Big picture + +Enable $[prodname] to advertise Kubernetes service IPs outside a cluster. $[prodname] supports advertising a service’s cluster IPs and external IPs. + +## Value + +Typically, Kubernetes service cluster IPs are accessible only within the cluster, so external access to the service requires a dedicated load balancer or ingress controller. In cases where a service’s cluster IP is not routable, the service can be accessed using its external IP. + +Just as $[prodname] supports advertising **pod IPs** over BGP, it also supports advertising Kubernetes **service IPs** outside a cluster over BGP. This avoids the need for a dedicated load balancer. This feature also supports equal cost multi-path (ECMP) load balancing across nodes in the cluster, as well as source IP address preservation for local services when you need more control. + +## Concepts + +### BGP makes it easy + +In Kubernetes, all requests for a service are redirected to an appropriate endpoint (pod) backing that service. Because $[prodname] uses BGP, external traffic can be routed directly to Kubernetes services by advertising Kubernetes service IPs into the BGP network. + +If your deployment is configured to peer with BGP routers outside the cluster, those routers (plus any other upstream places the routers propagate to) can send traffic to a Kubernetes service IP for routing to one of the available endpoints for that service. + +### Advertising service IPs: quick glance + +$[prodname] implements the Kubernetes **externalTrafficPolicy** using kube-proxy to direct incoming traffic to a correct pod. Advertisement is handled differently based on the service type that you configure for your service. + +| **Service mode** | **Cluster IP advertisement** | **Traffic is...** | Source IP address is... | +| ----------------- | ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------- | +| Cluster (default) | All nodes in the cluster statically advertise a route to the service CIDR. | Load balanced across nodes in the cluster using ECMP, then forwarded to appropriate pod in the service using SNAT. May incur second hop to another node, but good overall load balancing. | Obscured by SNAT | +| Local | The nodes with a pod backing the service advertise a specific route (/32 or /128) to the service's IP. | Load balanced across nodes with endpoints for the service. Avoids second hop for LoadBalancer and NodePort type services, traffic may be unevenly load balanced. (Other traffic is load balanced across nodes in the cluster.) | Preserved | + +If your $[prodname] deployment is configured to peer with BGP routers outside the cluster, those routers - plus any further upstream places that those routers propagate to - will be able to send traffic to a Kubernetes service cluster IP, and that traffic is routed to one of the available endpoints for that service. + +### Tips for success + +- Generally, we recommend using “Local” for the following reasons: + - If any of your network policy uses rules to match by specific source IP addresses, using Local is the obvious choice because the source IP address is not altered, and the policy will still work. + - Return traffic is routed directly to the source IP because “Local” services do not require undoing the source NAT (unlike “Cluster” services). +- Cluster IP advertisement works best with a ToR that supports ECMP. Otherwise, all traffic for a given route is directed to a single node. + +## Before you begin... + +**Required** + +- Calico CNI +- [Configure BGP peering](bgp.mdx) between $[prodname] and your network infrastructure +- For ECMP load balancing to services, the upstream routers must be configured to use BGP multipath. +- You need at least one external node outside the cluster that acts as a router, route reflector, or ToR that is peered with calico nodes inside the cluster. +- Services must be configured with the correct service mode (“Cluster” or “Local”) for your implementation. For `externalTrafficPolicy: Local`, the service must be type `LoadBalancer` or `NodePort`. + +**Limitations** + +- Supported in EKS and AWS, but only if you are using Calico CNI +- OpenShift, versions 4.5 and 4.6 + There is a [bug](https://github.com/kubernetes/kubernetes/issues/91374) where the source IP is not preserved by NodePort services or traffic via a Service ExternalIP with externalTrafficPolicy:Local. + + OpenShift users on v4.5 or v4.6 can use this [workaround to avoid SNAT with ExternalIP](https://docs.openshift.com/container-platform/4.7/nodes/clusters/nodes-cluster-enabling-features.html): + + ``` + oc edit featuregates.config.openshift.io cluster + spec: + customNoUpgrade: + enabled: + - ExternalPolicyForExternalIP + ``` + + Kubernetes users on version v1.18 or v1.19 can enable source IP preservation for NodePort services using the ExternalPolicyForExternalIP feature gate. + + Source IP preservation for NodePort and services and ExternalIPs is enabled by default in OpenShift v4.7+, and Kubernetes v1.20+. + +## How to + +- [Advertise service cluster IP addresses](#advertise-service-cluster-ip-addresses) +- [Advertise service external IP addresses](#advertise-service-external-ip-addresses) +- [Advertise service load balancer IP addresses](#advertise-service-load-balancer-ip-addresses) +- [Exclude certain nodes from advertisement](#exclude-certain-nodes-from-advertisement) + +### Advertise service cluster IP addresses + +1. Determine the service cluster IP range. (Or ranges, if your cluster is [dual stack](../ipam/ipv6.mdx).) + + The range(s) for your cluster can be inferred from the `--service-cluster-ip-range` option passed to the Kubernetes API server. For help, see the [Kubernetes API server reference guide](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/). + +1. Check to see if you have a default BGPConfiguration. + + ```bash + kubectl get bgpconfiguration.projectcalico.org default + ``` + +1. Based on above results, update or create a BGPConfiguration. + + **Update default BGPConfiguration**. + + Patch the BGPConfiguration using the following command, using your own service cluster IP CIDR in place of "10.0.0.0/24": + + ```bash + kubectl patch bgpconfiguration.projectcalico.org default -p '{"spec":{"serviceClusterIPs": [{"cidr": "10.0.0.0/24"}]}}' + ``` + + **Create default BGPConfiguration**. + + Use the following sample command to create a default BGPConfiguration. Add your CIDR blocks, covering the cluster IPs to be advertised, in the `serviceClusterIPs` field, for example: + + ```bash + kubectl create -f - < + asNumber: + filters: + - export-to-tor +``` + +### Configure BGP peering with workloads + +In the parent cluster, create a `BGPPeer` resource to peer with the workloads. + +- The `localWorkloadSelector` field is used to select which workloads to peer with. + In this example, we select workloads with the label `color=red`. + +- Include the `BGPFilter` resources defined above in the `BGPPeer` resource to control + which routes are imported and exported. + +```yaml +apiVersion: projectcalico.org/v3 +kind: BGPPeer +metadata: + name: peer-to-workloads +spec: + localWorkloadSelector: color == 'red' + asNumber: + filters: + - no-export + - accept-nested-ip-pools +``` + +Both per-node and global `BGPPeer` resources are supported. To limit the peering to certain nodes, use the `nodeSelector` field. + +:::note + +`localWorkloadSelector` is scoped to all workloads and it does not currently have a corresponding +"namespace selector". To match on namespace labels, you can prefix the label name with `pcns.`. + +::: + +In the child cluster, + +- Ensure that the node-to-node mesh is disabled in the `BGPConfiguration` resource and + that the workload cluster's AS number is set: + + ```yaml + apiVersion: projectcalico.org/v3 + kind: BGPConfiguration + metadata: + name: default + spec: + nodeToNodeMeshEnabled: false + asNumber: + ``` + +- Create a `BGPPeer` resource (per desired IP version) that peers with the + parent clusters' virtual IP: + + ```yaml + apiVersion: projectcalico.org/v3 + kind: BGPPeer + metadata: + name: peer-to-parent-cluster + spec: + peerIP: 169.254.0.179 + asNumber: + --- + apiVersion: projectcalico.org/v3 + kind: BGPPeer + metadata: + name: peer-to-parent-cluster-v6 + spec: + peerIP: "fdc9:9723:09bc::1" + asNumber: + ``` + +### Configure more than one nested cluster + +You can configure more than one nested cluster by: + +- Choosing a unique non-overlapping IP pool for each nested cluster. +- In the parent cluster: creating a new import `BGPFilter` resource for each cluster to import its routes. +- In the parent cluster: creating a new, uniquely-named `BGPPeer` resource for each cluster to peer with the + parent cluster. You can re-use the shared "no-export" `BGPFilter` resource with every nested cluster peering. +- In the parent cluster: updating the `export-to-tor` `BGPFilter` resource to include the new cluster's CIDRs. +- In the nested cluster: adding a copy of the `peer-to-parent-cluster` `BGPPeer`(s). These should + be the same in all nested clusters since a parent cluster always uses the same virtual IP for all + nested clusters. + +:::note + +If you plan to use many nested clusters consider allocating their IP pools from a single larger +CIDR. Then the `export-to-tor` filter can be simplified to match on the larger CIDR only. + +If you trust the nested clusters, similarly, you could also re-use a single `accept-nested-ip-pools` filter +for all child clusters. + +::: + +## Additional resources + +- [BGP guide](./bgp.mdx) +- [BGP L3 interconnect guide](../../reference/architecture/design/l3-interconnect-fabric.mdx) +- [Node resource](../../reference/resources/node.mdx) +- [BGP configuration resource](../../reference/resources/bgpconfig.mdx) +- [BGP peer resource](../../reference/resources/bgppeer.mdx) +- [BGP filter resource](../../reference/resources/bgpfilter.mdx) + diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/bgp.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/bgp.mdx new file mode 100644 index 0000000000..f419958a08 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/bgp.mdx @@ -0,0 +1,364 @@ +--- +description: Set up BGP peering for Calico Cloud connected clusters — full mesh, per-node peers, top-of-rack switches, and route reflectors — with BGPPeer and BGPConfiguration resources. +--- + +# Configure BGP peering + +## Big picture + +Configure BGP (Border Gateway Protocol) between Calico nodes or peering with network infrastructure to distribute routing information. + +## Value + +$[prodname] nodes can exchange routing information over BGP to enable reachability for $[prodname] networked workloads (Kubernetes pods or OpenStack VMs). In an on-premises deployment this allows you to make your workloads first-class citizens across the rest of your network. In public cloud deployments, it provides an efficient way of distributing routing information within your cluster, and is often used in conjunction with IPIP overlay or cross-subnet modes. + +## Concepts + +### BGP + +**BGP** is a standard protocol for exchanging routing information between routers in a network. Each router running BGP has one or more **BGP peers** - other routers which they are communicating with over BGP. You can think of $[prodname] networking as providing a virtual router on each of your nodes. You can configure $[prodname] nodes to peer with each other, with route reflectors, or with top-of-rack (ToR) routers. + +### Common BGP topologies + +There are many ways to configure a BGP network depending on your environment. Here are some common ways it is done with $[prodname]. + +### Full-mesh + +When BGP is enabled, $[prodname]’s default behavior is to create a **full-mesh** of internal BGP (iBGP) connections where each node peers with each other. This allows $[prodname] to operate over any L2 network, whether public cloud or private cloud, or, if IPIP is [configured](vxlan-ipip.mdx), to operate as an overlay over any network that does not block IPIP traffic. $[prodname] does not use BGP for VXLAN overlays. + +:::note + +Most public clouds support IPIP. The notable exception is Azure, which blocks IPIP traffic. So if you want to run Calico as an overlay network in Azure, you must [configure $[prodname] to use VXLAN](vxlan-ipip.mdx). + +::: + +Full-mesh works great for small and medium-size deployments of say 100 nodes or less, but at significantly larger scales full-mesh becomes less efficient, and we recommend using route reflectors. + +### Route reflectors + +To build large clusters of internal BGP (iBGP), **BGP route reflectors** can be used to reduce the number of BGP peerings used on each node. In this model, some nodes act as route reflectors and are configured to establish a full mesh amongst themselves. Other nodes are then configured to peer with a subset of those route reflectors (typically 2 for redundancy), reducing the total number BGP peering connections compared to full-mesh. + +### Top of Rack (ToR) + +In **on-premises deployments**, you can configure $[prodname] to peer directly with your physical network infrastructure. Typically, this involves disabling $[prodname]’s default full-mesh behavior, and instead peer $[prodname] with your L3 ToR routers. There are many ways to build an on-premises BGP network. How you configure your BGP is up to you - $[prodname] works well with both iBGP and eBGP configurations, and you can effectively treat $[prodname] like any other router in your network design. + +Depending on your topology, you may also consider using BGP route reflectors within each rack. However, this is typically needed only if the number of nodes in each L2 domain is large (> 100). + +For a deeper look at common on-premises deployment models, see [Calico over IP Fabrics](../../reference/architecture/design/l2-interconnect-fabric.mdx). + +## Before you begin... + +**Required** + +- Calico CNI + +{/*//TODO-CC-XREF- [calicoctl] (/operations/clis/calicoctl/install) must be installed and configured*/} + +## How to + +:::note + +Significantly changing $[prodname]'s BGP topology, such as changing from full-mesh to peering with ToRs, may result in temporary loss of pod network connectivity during the reconfiguration process. It is recommended to only make such changes during a maintenance window. + +::: + +- [Configure a global BGP peer](#configure-a-global-bgp-peer) +- [Configure a per-node BGP peer](#configure-a-per-node-bgp-peer) +- [Configure a node to act as a route reflector](#configure-a-node-to-act-as-a-route-reflector) +- [Disable the default BGP node-to-node mesh](#disable-the-default-bgp-node-to-node-mesh) +- [Change from node-to-node mesh to route reflectors without any traffic disruption](#change-from-node-to-node-mesh-to-route-reflectors-without-any-traffic-disruption) +- [View BGP peering status for a node](#view-bgp-peering-status-for-a-node) +- [View BGP info on all peers for a node](#view-bgp-info-on-all-peers-for-a-node) +- [Change the default global AS number](#change-the-default-global-as-number) +- [Change AS number for a particular node](#change-as-number-for-a-particular-node) +- [Set a local AS number for a specific BGP peer](#set-a-local-as-number-for-a-specific-bgp-peer) +- [Configure a BGP filter](#configure-a-bgp-filter) +- [Configure a BGP peer with a BGP filter](#configure-a-bgp-peer-with-a-bgp-filter) + +### Configure a global BGP peer + +Global BGP peers apply to all nodes in your cluster. This is useful if your network topology includes BGP speakers that will be peered with every $[prodname] node in your deployment. + +The following example creates a global BGP peer that configures every $[prodname] node to peer with **192.20.30.40** in AS **64567**. + +```yaml +apiVersion: projectcalico.org/v3 +kind: BGPPeer +metadata: + name: my-global-peer +spec: + peerIP: 192.20.30.40 + asNumber: 64567 +``` + +### Configure a per-node BGP peer + +Per-node BGP peers apply to one or more nodes in the cluster. You can choose which nodes by specifying the node’s name exactly, or using a label selector. + +The following example creates a BGPPeer that configures every $[prodname] node with the label, **rack: rack-1** to peer with **192.20.30.40** in AS **64567**. + +```yaml +apiVersion: projectcalico.org/v3 +kind: BGPPeer +metadata: + name: rack1-tor +spec: + peerIP: 192.20.30.40 + asNumber: 64567 + nodeSelector: rack == 'rack-1' +``` + +### Configure a node to act as a route reflector + +$[prodname] nodes can be configured to act as route reflectors. To do this, each node that you want to act as a route reflector must have a cluster ID - typically an unused IPv4 address. + +To configure a node to be a route reflector with cluster ID 244.0.0.1, run the following command. + +```bash +kubectl annotate node my-node projectcalico.org/RouteReflectorClusterID=244.0.0.1 +``` + +Typically, you will want to label this node to indicate that it is a route reflector, allowing it to be easily selected by a BGPPeer resource. You can do this with kubectl. For example: + +```bash +kubectl label node my-node route-reflector=true +``` + +Now it is easy to configure route reflector nodes to peer with each other and other non-route-reflector nodes using label selectors. For example: + +```yaml +kind: BGPPeer +apiVersion: projectcalico.org/v3 +metadata: + name: peer-with-route-reflectors +spec: + nodeSelector: all() + peerSelector: route-reflector == 'true' +``` + +:::note + +Adding `routeReflectorClusterID` to a node spec will remove it from the node-to-node mesh immediately, tearing down the +existing BGP sessions. Adding the BGP peering will bring up new BGP sessions. This will cause a short (about 2 seconds) +disruption to data plane traffic of workloads running in the nodes where this happens. To avoid this, make sure no +workloads are running on the nodes, by provisioning new nodes or by running `kubectl drain` on the node (which may +itself cause a disruption as workloads are drained). + +::: + +### Disable the default BGP node-to-node mesh + +The default **node-to-node BGP mesh** may be turned off to enable other BGP topologies. To do this, modify the default **BGP configuration** resource. + +Run the following command to disable the BGP full-mesh: + +```bash +calicoctl patch bgpconfiguration default -p '{"spec": {"nodeToNodeMeshEnabled": false}}' +``` + +:::note + +If the default BGP configuration resource does not exist, you need to create it first. See [BGP configuration](../../reference/resources/bgpconfig.mdx) for more information. + +::: + +:::note + +Disabling the node-to-node mesh will break pod networking until/unless you configure replacement BGP peerings using BGPPeer resources. +You may configure the BGPPeer resources before disabling the node-to-node mesh to avoid pod networking breakage. + +::: + +### Change from node-to-node mesh to route reflectors without any traffic disruption + +Switching from node-to-node BGP mesh to BGP route reflectors involves tearing down BGP sessions and bringing up new ones. This causes a short +data plane network disruption (of about 2 seconds) for workloads running on the nodes in the cluster. To avoid this, you may provision +route reflector nodes and bring their BGP sessions up before tearing down the node-to-node mesh sessions. + +Follow these steps to do so: + +1. [Provision new nodes to be route reflectors.](#configure-a-node-to-act-as-a-route-reflector) The nodes [should not be schedulable](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) + and they should have `routeReflectorClusterID` in their spec. These won't be part of the existing + node-to-node BGP mesh, and will be the route reflectors when the mesh is disabled. These nodes should also have a label like + `route-reflector` to select them for the BGP peerings. [Alternatively](https://kubernetes.io/docs/tasks/administer-cluster/safely-drain-node/), + you can drain workloads from existing nodes in your cluster by running `kubectl drain ` to configure them to be route reflectors, + but this will cause a disruption on the workloads on those nodes as they are drained. + +2. Also set up a [BGPPeer](#configure-a-node-to-act-as-a-route-reflector) spec to configure route reflector nodes to peer with each other and other non-route-reflector nodes + using label selectors. + +3. Wait for these peerings to be established. This can be [verified](#view-bgp-peering-status-for-a-node) by running `sudo calicoctl node status` on the nodes. Alternatively, you can create a [`CalicoNodeStatus` resource](../../reference/resources/caliconodestatus.mdx) to get BGP session status for the node. + +4. [Disable the BGP node-to-node mesh for the cluster.](#disable-the-default-bgp-node-to-node-mesh) + +5. If you did drain workloads from the nodes or created them as unschedulable, mark the nodes as schedulable again (e.g. by running `kubectl uncordon `). + +### View BGP peering status for a node + +Create a [CalicoNodeStatus resource](../../reference/resources/caliconodestatus.mdx) to monitor BGP session status for the node. + +Alternatively, you can run the `calicoctl node status` command on a given node to learn more about its BGP status. + +:::note + +This command communicates with the local $[prodname] agent, so you must execute it on the node whose status you are attempting to view. + +::: + +### View BGP info on all peers for a node + +You can use `calicoctl` to view the BGP information for all peers of a particular node, including connection status, routing statistics, and BGP state. This is useful for confirming that your configuration is behaving as desired, and for more detailed troubleshooting. + +Run the following command from anywhere you have access to `kubectl`: + +```bash +calicoctl bgp peers +``` + +Where `` is the resource name for one of the Calico node pods within your cluster. + +### Change the default global AS number + +By default, all Calico nodes use the 64512 autonomous system, unless a per-node AS has been specified for the node. You can change the global default for all nodes by modifying the default **BGPConfiguration** resource. The following example command sets the global default AS number to **64513**. + +```bash +kubectl patch bgpconfiguration default -p '{"spec": {"asNumber": "64513"}}' +``` + +:::note + +If the default BGP configuration resource does not exist, you need to create it first. See [BGP configuration](../../reference/resources/bgpconfig.mdx) for more information. + +::: + +### Change AS number for a particular node + +You can configure an AS for a particular node by modifying the node object using `calicoctl`. For example, the following command changes the node named **node-1** to belong to **AS 64514**. + +```bash +calicoctl patch node node-1 -p '{"spec": {"bgp": {"asNumber": "64514"}}}' +``` + +### Set a local AS number for a specific BGP peer + +You can configure a distinct local AS number for an individual BGP peer by modifying the `BGPPeer` resource using `calicoctl` or `kubectl`. This overrides the node-wide or global AS number defined in the `BGPConfiguration`, allowing different local ASNs to be used on a per-peer basis. If `localASNumber` is not set, Calico will fall back to the node-wide or global AS number. + +```bash +kubectl patch bgppeer -p '{"spec": {"localASNumber": 64577}}' +``` + +### Configure a BGP filter + +BGP filters control which routes are imported and exported between BGP peers. + +The BGP filter rules (importVX, exportVX) are applied sequentially, taking the +`action` of the first matching rule. When no rules are matched, the default +`action` is `Accept`. + +In order for a BGPFilter to be used in a BGP peering, its `name` +must be added to `filters` of the corresponding BGPPeer resource. + +The following example creates a BGPFilter + +```yaml +apiVersion: projectcalico.org/v3 +kind: BGPFilter +metadata: + name: my-filter +spec: + exportV4: + - action: Accept + matchOperator: In + cidr: 77.0.0.0/16 + - action: Reject + source: RemotePeers + - action: Reject + interface: '*.calico' + importV4: + - action: Reject + matchOperator: NotIn + cidr: 44.0.0.0/16 + exportV6: + - action: Reject + source: RemotePeers + - action: Reject + interface: '*.calico' + importV6: + - action: Accept + matchOperator: Equal + cidr: 5000::0/64 + - action: Reject +``` + +### Configure a BGP peer with a BGP filter + +BGP peers can use BGP filters to control which routes are imported or exported between them. + +The following example creates a BGPFilter and associates it with a BGPPeer +:::note + +BGPFilters are applied in the order listed on a BGPPeer + +::: + +```yaml +kind: BGPFilter +apiVersion: projectcalico.org/v3 +metadata: + name: first-bgp-filter +spec: + exportV4: + - action: Accept + matchOperator: In + cidr: 77.0.0.0/16 + source: RemotePeers + importV4: + - action: Reject + matchOperator: NotIn + cidr: 44.0.0.0/16 + exportV6: + - action: Reject + interface: '*.calico' + importV6: + - action: Accept + matchOperator: Equal + cidr: 5000::0/64 +--- +kind: BGPFilter +apiVersion: projectcalico.org/v3 +metadata: + name: second-bgp-filter +spec: + exportV4: + - action: Accept + matchOperator: In + cidr: 77.0.0.0/16 + interface: '*.calico' + importV4: + - action: Reject + matchOperator: NotIn + cidr: 44.0.0.0/16 + exportV6: + - action: Reject + source: RemotePeers + importV6: + - action: Reject +--- +kind: BGPPeer +apiVersion: projectcalico.org/v3 +metadata: + name: peer-with-filter +spec: + peerSelector: has(filter-bgp) + filters: + - first-bgp-filter + - second-bgp-filter +``` + +## Additional resources + +- [Node resource](../../reference/resources/node.mdx) +- [Configure BGP peering with nested clusters running on Kubevirt VMs](./bgp-to-workload.mdx) +- [BGP configuration resource](../../reference/resources/bgpconfig.mdx) +- [BGP peer resource](../../reference/resources/bgppeer.mdx) +- [BGP filter resource](../../reference/resources/bgpfilter.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/custom-bgp-config.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/custom-bgp-config.mdx new file mode 100644 index 0000000000..3f37c0275a --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/custom-bgp-config.mdx @@ -0,0 +1,49 @@ +--- +description: Override the default BIRD BGP templates in a Calico Cloud connected cluster to access advanced BIRD features for proof-of-concept and special-case routing. +--- + +# Custom BGP Configuration + +## Big picture + +Use customized BIRD configuration files to enable specialized use-cases. + +## Concepts + +In $[prodname], BGP is handled by [BIRD](https://github.com/projectcalico/bird). +The BIRD configurations are templated out through [confd](https://github.com/projectcalico/confd). +You can modify the BIRD configuration to use BIRD features which are not typically exposed using the +default configuration provided with $[prodname]. + +Customization of BGP templates should be done only with the help of your Tigera Support representative. + +## Before you begin + +**Required** + +- Calico CNI + +## How to + +- [Update BGP configuration](#update-bgp-configuration) +- [Apply BGP customizations](#apply-bgp-customizations) based on how you've deployed $[prodname]: + +### Update BGP configuration + +Using the directions provided with the templates, set the correct values +for the BGP configuration using these resources: + +- [BGP Configuration](../../reference/resources/bgpconfig.mdx) +- [BGP Peer](../../reference/resources/bgppeer.mdx) +{/* //TODO-CC-XREFS - [calicoctl](/reference/clis/calicoctl/index)*/} + +### Apply BGP Customizations + +1. Create your confd templates. +1. Create a ConfigMap from the templates. + +``` +kubectl create configmap bird-templates -n tigera-operator --from-file= +``` + +The created config map will be used to populate the $[prodname] BIRD configuration file templates. If a template with the same name already exists within the node container, it will be overwritten with the contents from the config map. diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/dual-tor.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/dual-tor.mdx new file mode 100644 index 0000000000..b974e55a53 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/dual-tor.mdx @@ -0,0 +1,642 @@ +--- +description: Deploy a dual ToR cluster connected to Calico Cloud so two independent connectivity planes provide redundancy between racks for on-premises clusters. +--- + +# Deploy a dual ToR cluster + +## Big picture + +Deploy a dual plane cluster to provide redundant connectivity between your workloads for on-premises deployments. + +:::note + +Dual ToR is not supported if you are using BGP with encapsulation (VXLAN or IP-in-IP). + +::: + +## Value + +A dual plane cluster provides two independent planes of connectivity between all cluster +nodes. If a link or software component breaks somewhere in one of those planes, cluster +nodes can still communicate over the other plane, and the cluster as a whole continues to +operate normally. + +## Concepts + +### Dual plane connectivity, aka "dual ToR" + +Large on-prem Kubernetes clusters, split across multiple server racks, can use two or more +independent planes of connectivity between all the racks. The advantages are: + +- The cluster can still function, even if there is a single break in connectivity + somewhere. + +- The cluster can load balance across the bandwidth of _both_ planes, when both planes + are available. + +The redundant approach can be applied within each rack as well, such that each node has +two or more independent connections to those connectivity planes. Typically, each rack +has two top-of-rack routers ("ToRs") and each node has two fabric-facing interfaces, each +of which connects over a separate link or Ethernet to one of the ToRs for the rack. + +Here's an example of how a dual plane setup might look, with just two racks and two nodes +in each rack. For simplicity, we've shown the connections _between_ racks as single +links; in reality that would be more complex, but still following the overall dual plane +paradigm. + +![dual-tor](/img/calico-enterprise/dual-tor.png) + +Because of the two ToRs per rack, the whole setup is often referred to as "dual ToR". + +### Network design for a dual ToR cluster + +For a dual ToR cluster to operate seamlessly when there is a break on one of the planes, +several things are needed. + +- Each node should have a stable IP address that is independent of its per-interface + addresses and remains valid if the connectivity through _one_ of those interfaces goes + down. + +- Each node must somehow know or learn the stable IP address of every other node. + +- Wherever a connection (other than BGP) is to or from a _node_ (as opposed to a + non-host-networked pod), that connection should use the node's stable address as its + destination or source IP (respectively), so that the connection can continue working if + one of the planes has an outage. + +- Importantly, this includes connections that Kubernetes uses as part of its own control + plane, such as between the Kubernetes API server and kubelet on each node. Ideally, + therefore, the stable IP address setup on each node should happen *before* Kubernetes + starts running. + +- BGP is an exception to the previous points - in fact, the _only_ exception - because we + want each node's BGP peerings to be interface-specific and to reflect what is actually + reachable, moment by moment, over that interface. The Linux routing table then + automatically adjusts so that the route to each remote destination is either ECMP - + when both planes are up - or non-ECMP when it can only be reached over one of the + planes. + +- BGP peerings should be configured to detect any outages, and to propagate their + consequences, as quickly as possible, so that the routing can quickly respond on each + node. Note that this is quite different from the reasoning for a single connectivity + plane, where it's better to delay any network churn, on the assumption that an outage + will be quickly fixed. + +Finally, to spread load evenly and maximise use of both planes, when both available, the +routers and Linux kernel need to be configured for efficient ECMP. + +### Calico's early networking architecture + +$[prodname]'s $[nodecontainer] image can be run in an "early networking" mode, +on each node, to perform all of the above points that are needed before Kubernetes starts +running. That means that it: + +- Provisions the stable IP address, and assigns it to the loopback interface. + +- Ensures the stable address will be used as the source + IP for any outgoing connections, by adjusting the scope of interface addresses. + +- Starts running BGP, peering with the node's ToRs, to advertise the node's + stable address to other nodes. + +- Configures efficient ECMP in the Linux kernel (with `fib_multipath_hash_policy=1` and + `fib_multipath_use_neigh=1`). + +More detail is given below on how to run this early networking image. A key point is that +it must run as soon as possible after each node boot, and before Kubernetes starts on the +node, so it is typically run as a Docker or podman container. + +After its start-of-day provisioning, the early networking container keeps running so that +it can tag-team the BGP role with Calico's regular BGP service running inside the +$[nodecontainer] _pod_: + +- Initially the $[nodecontainer] pod does not yet exist, so the early networking + BGP runs to advertise out the node's stable address. + +- After Kubernetes has started on the node, and Calico has been installed in Kubernetes, + the $[nodecontainer] pod runs and starts its own BGP service. The early + networking container spots that the regular BGP service is now running and so shuts + down its own BGP. Now the regular BGP service handles the advertisement of the stable + address, as well as pod IPs and so on. + +- Later, the $[nodecontainer] pod might be shut down, e.g. for restart or upgrade. + If the downtime continues for longer than the graceful restart period, the early + networking container spots this and restarts its own BGP, to ensure that the node's + stable IP address continues to be advertised to other nodes. The cycle can now repeat + from the "Initially" state above. + + :::note + + The default graceful restart period is 120s for traditional BGP GR and + 3600s for LLGR. + + ::: + +### BGP configuration for rapid outage detection + +A dual ToR cluster needs Calico BGPPeer resources to specify how each node should peer +with its ToRs. The remaining parts of the dual ToR network design are implemented as +properties of those BGP peerings, and as corresponding properties on the BGP configuration +between and within the ToRs and core infrastructure. + +Specifically, on Calico's BGPPeer resource, + +- the `failureDetectionMode` field is used to enable BFD + +- the `restartMode` field can be used to enable long-lived graceful restart (LLGR). + +See below for more on the benefits of these settings. When they are used, consistent +settings are needed on the ToRs and core infrastructure. + +### ECMP routing + +An "Equal Cost Multiple Path" (ECMP) route is one that has multiple possible ways to reach +a given destination or prefix, all of which are considered to be equally good. A dual ToR +setup naturally generates ECMP routes, with the different paths going over the different +connectivity planes. + +When using an ECMP route, Linux decides how to balance traffic across the available paths, +including whether this is informed by TCP and UDP port numbers as well as source and +destination IP addresses, whether the decision is made per-packet, per-connection, or in +some other way, and so on; and the details here have varied with Linux kernel version. +For a clear account of the exact options and behaviors for different kernel versions, +please see [this blog](https://web.archive.org/web/20210204031636/https://cumulusnetworks.com/blog/celebrating-ecmp-part-two/). + +### BFD + +Bidirectional Forwarding Detection (BFD) is [a protocol](https://tools.ietf.org/html/rfc5880) + that detects very quickly when forwarding +along a particular path stops working - whether that's because a link has broken +somewhere, or some software component along the path. + +In a dual ToR setup, rapid failure detection is important so that traffic flows within the +cluster can quickly adjust to using the other available connectivity plane. + +### Long lived graceful restart + +Long Lived Graceful Restart (LLGR) is [an extension for BGP](https://tools.ietf.org/html/draft-uttaro-idr-bgp-persistence-05) + that handles link +failure by lowering the preference of routes over that link. This is a compromise between +the base BGP behaviour - which is immediately to remove those routes - and traditional BGP +Graceful Restart behaviour - which is not to change those routes at all, until some +configured time has passed. + +For a dual ToR setup, LLGR is helpful, as explained in more detail by [this blog](https://vincent.bernat.ch/en/blog/2018-bgp-llgr) +, because: + +- If a link fails somewhere, the immediate preference lowering allows traffic to adjust + immediately to use the other connectivity plane. + +- If a node is restarted, we still get the traditional Graceful Restart behaviour whereby + routes to that node persist in the rest of the network. + + +## Before you begin + +**Unsupported** + +- AKS +- EKS +- GKE + +**Required** + +- Calico CNI + +## How to + +- [Prepare YAML resources describing the layout of your cluster](#prepare-yaml-resources-describing-the-layout-of-your-cluster) +- [Arrange for dual-homed nodes to run $[nodecontainer] on each boot](#arrange-for-dual-homed-nodes-to-run-cnx-node-on-each-boot) +- [Configure your ToR routers and infrastructure](#configure-your-tor-routers-and-infrastructure) +- [Install Kubernetes and $[prodname]](#install-kubernetes-and-calico-cloud) +- [Verify the deployment](#verify-the-deployment) + +### Prepare YAML resources describing the layout of your cluster + +1. Prepare BGPPeer resources to specify how each node in your cluster should peer with + the ToR routers in its rack. For example, if your rack 'A' has ToRs with IPs + 172.31.11.100 and 172.31.12.100 and the rack AS number is 65001: + + ```yaml + apiVersion: projectcalico.org/v3 + kind: BGPPeer + metadata: + name: ra1 + spec: + nodeSelector: "rack == 'ra' || rack == 'ra_single'" + peerIP: 172.31.11.100 + asNumber: 65001 + sourceAddress: None + --- + apiVersion: projectcalico.org/v3 + kind: BGPPeer + metadata: + name: ra2 + spec: + nodeSelector: "rack == 'ra'" + peerIP: 172.31.12.100 + asNumber: 65001 + sourceAddress: None + ``` + + :::note + + The effect of the `nodeSelector` fields here is that any node with label + `rack: ra` will peer with both these ToRs, while any node with label `rack: ra_single` will peer with only the first ToR. For optimal dual ToR function and + resilience, nodes in rack 'A' should be labelled `rack: ra`, but `rack: ra_single` + can be used instead on any nodes which cannot be dual-homed. + + ::: + + Repeat for as many racks as there are in your cluster. Each rack needs a new pair of + BGPPeer resources with its own ToR addresses and AS number, and `nodeSelector` fields + matching the nodes that should peer with its ToR routers. + + Depending on what your ToR supports, consider also setting these fields in each + BGPPeer: + + - `failureDetectionMode: BFDIfDirectlyConnected` to enable BFD, when possible, for + fast failure detection. + + :::note + + $[prodname] only supports BFD on directly connected peerings, but + in practice nodes are normally directly connected to their ToRs. + + ::: + + - `restartMode: LongLivedGracefulRestart` to enable LLGR handling when the node needs + to be restarted, if your ToR routers support LLGR. If not, we recommend instead + `maxRestartTime: 10s`. + + - `birdGatewayMode: DirectIfDirectlyConnected` to enable the "direct" next hop + algorithm, if that is helpful for optimal interworking with your ToR routers. + + :::note + + For directly connected BGP peerings, BIRD provides two gateway + computation modes, "direct" and "recursive". + + "recursive" is the default, but "direct" can give better results when the ToR + also acts as the route reflector (RR) for the rack. + Specifically, a combined ToR/RR should ideally keep the BGP next hop intact (aka + "next hop keep") when reflecting routes from other nodes in the same rack, but + add itself as the BGP next hop (aka "next hop self") when forwarding routes from + outside the rack. If your ToRs can be configured to do that, fine. + If they cannot, an effective workaround is to configure the ToRs to do "next hop + keep" for all routes, with "gateway direct" on the $[prodname] nodes. In + effect the “gateway direct” applies a “next hop self” when needed, but otherwise + not. + + ::: + +1. Prepare this BGPConfiguration resource to [disable the full node-to-node mesh](bgp.mdx#disable-the-default-bgp-node-to-node-mesh): + + ```yaml + apiVersion: projectcalico.org/v3 + kind: BGPConfiguration + metadata: + name: default + spec: + nodeToNodeMeshEnabled: false + ``` + +1. Prepare disabled IPPool resources for the CIDRs from which you will allocate stable + addresses for dual-homed nodes. For example, if the nodes in rack 'A' will have + stable addresses from 172.31.10.0/24: + + ```yaml + apiVersion: projectcalico.org/v3 + kind: IPPool + metadata: + name: ra-stable + spec: + cidr: 172.31.10.0/24 + disabled: true + nodeSelector: all() + ``` + + If the next rack uses a different CIDR, define a similar IPPool for that rack, and so + on. + + :::note + + These IPPool definitions tell $[prodname]'s BGP component to export + routes within the given CIDRs, which is essential for the core BGP infrastructure to + learn how to route to each stable address. `disabled: true` tells $[prodname] + _not_ to use these CIDRs for pod IPs. + + ::: + +1. Prepare an enabled IPPool resource for your default CIDR for pod IPs. For example: + + ```yaml + apiVersion: projectcalico.org/v3 + kind: IPPool + metadata: + name: default-ipv4 + spec: + cidr: 10.244.0.0/16 + nodeSelector: all() + ``` + + :::note + + The CIDR must match what you specify elsewhere in the Kubernetes + installation. For example, `networking.clusterNetwork.cidr` in OpenShift's install + config, or `--pod-network-cidr` with kubeadm. You should not specify `ipipMode` or + `vxlanMode`, as these are incompatible with dual ToR operation. `natOutgoing` can + be omitted, as here, if your core infrastructure will perform an SNAT for traffic + from pods to the Internet. + + ::: + +1. Prepare an EarlyNetworkConfiguration resource to specify the additional information + that is needed for each node in a multi-rack dual ToR cluster: + + - The stable address for the node. + - Its BGP AS number. + - The IPs that the node should peer with, when $[nodecontainer] runs + as a container for early networking setup after each node boot. + - Any labels that the node should have, so as to match the right BGPPeer definitions + for its rack, when $[nodecontainer] runs as a Kubernetes pod. + +
+ With OpenShift, also add a toplevel `platform: openshift` setting. + + :::note + + `platform: openshift` triggers additional per-node setup that is needed + during OpenShift's bootstrapping phase. + + ::: + + For example, with IP addresses and AS numbers similar as for other resources above: + + ```yaml noValidation + apiVersion: projectcalico.org/v3 + kind: EarlyNetworkConfiguration + spec: + platform: openshift + nodes: + # worker1 + - interfaceAddresses: + - 172.31.11.3 + - 172.31.12.3 + stableAddress: + address: 172.31.10.3 + asNumber: 65001 + peerings: + - peerIP: 172.31.11.100 + - peerIP: 172.31.12.100 + labels: + rack: ra + # worker2 + - interfaceAddresses: + - 172.31.21.4 + - 172.31.22.4 + stableAddress: + address: 172.31.20.4 + asNumber: 65002 + peerings: + - peerIP: 172.31.21.100 + - peerIP: 172.31.22.100 + labels: + rack: rb + ... + ``` + +1. Prepare a ConfigMap resource named "bgp-layout", in namespace "tigera-operator", that + wraps the EarlyNetworkConfiguration like this: + + ```yaml noValidation + apiVersion: v1 + kind: ConfigMap + metadata: + name: bgp-layout + namespace: tigera-operator + data: + earlyNetworkConfiguration: | + apiVersion: projectcalico.org/v3 + kind: EarlyNetworkConfiguration + spec: + nodes: + # worker1 + - interfaceAddresses: + ... + ``` + +:::note + +EarlyNetworkConfiguration supplies labels and AS numbers to apply to each +Calico node, as well as peering and other network configuration to use during node +startup prior to receiving BGPPeer and BGPConfiguration resources from the datastore. +EarlyNetworkConfiguration will be superseded by any BGPPeer or BGPConfiguration +resources after successful startup. + +::: + +### Arrange for dual-homed nodes to run $[nodecontainer] on each boot + +$[prodname]'s $[nodecontainer] image normally runs as a Kubernetes pod, but +for dual ToR setup it must also run as a container after each boot of a dual-homed node. +For example: + +``` +podman run --privileged --net=host \ + -v /calico-early:/calico-early -e CALICO_EARLY_NETWORKING=/calico-early/cfg.yaml \ + $[registry]$[imageNames.node]:latest +``` + +The environment variable `CALICO_EARLY_NETWORKING` must point to the +EarlyNetworkConfiguration prepared above, so that EarlyNetworkConfiguration YAML must be +copied into a file on the node (here, `/calico-early/cfg.yaml`) and mapped into the +$[nodecontainer] container. + +We recommend defining systemd services to ensure that early networking runs on each boot, +and before kubelet starts on the node. Following is an example that may need tweaking for +your particular platform, but that illustrates the important points. + +Firstly, a "calico-early" service that runs the Calico early networking on each boot: + +``` +[Unit] +Wants=network-online.target +After=network-online.target +After=nodeip-configuration.service +[Service] +ExecStartPre=/bin/sh -c "rm -f /etc/systemd/system/kubelet.service.d/20-nodenet.conf /etc/systemd/system/crio.service.d/20-nodenet.conf; systemctl daemon-reload" +ExecStartPre=-/bin/podman rm -f calico-early +ExecStartPre=/bin/mkdir -p /etc/calico-early +ExecStartPre=/bin/sh -c "test -f /etc/calico-early/details.yaml || /bin/curl -o /etc/calico-early/details.yaml http://172.31.1.1:8080/calico-early/details.yaml" +ExecStart=/bin/podman run --rm --privileged --net=host --name=calico-early -v /etc/calico-early:/etc/calico-early -e CALICO_EARLY_NETWORKING=/etc/calico-early/details.yaml $[registry]$[imageNames.node]:latest +[Install] +WantedBy=multi-user.target +``` + +:::note + +- You must also install your Tigera-issued pull secret at `/root/.docker/config.json`, + on each node, to enable pulling from $[registry]. +- Some OpenShift versions have a `nodeip-configuration` service that configures + kubelet's `--node-ip` option **wrongly** for a dual ToR setup. The + `After=nodeip-configuration.service` setting and the deletion of `20-nodenet.conf` + undo that service's work so that kubelet can choose its own IP correctly (using a + reverse DNS lookup). +- The `/bin/curl ...` line shows how you can download the EarlyNetworkConfiguration + YAML from a central hosting point within your cluster. + +::: + +Secondly, a "calico-early-wait" service that delays kubelet until after the Calico early +networking setup is in place: + +``` +[Unit] +After=calico-early.service +Before=kubelet.service +[Service] +Type=oneshot +ExecStart=/bin/sh -c "while sleep 5; do grep -q 00000000:1FF3 /proc/net/tcp && break; done; sleep 15" +[Install] +WantedBy=multi-user.target +``` + +:::note + +- The `ExecStart` line here arranges that kubelet will not start running until the + calico-early service has started listening on port 8179 (hex `1FF3`). 8179 is the + port that the calico-early service uses for pre-Kubernetes BGP. +- We have sometimes observed issues if kubelet starts immediately after Calico's early + networking setup, because of NetworkManager toggling the hostname. The final `sleep 15` allows for such changes to settle down before kubelet starts. + +::: + +On OpenShift you should wrap the above service definitions in `MachineConfig` resources +for the control and worker nodes. + +On other platforms either define and enable the above services directly, or use +whatever API the platform provides to define and enable services on new nodes. + +### Configure your ToR routers and infrastructure + +You should configure your ToR routers to accept all the BGP peerings from +$[prodname] nodes, to reflect routes between the nodes in that rack, and to +propagate routes between the ToR routers in different racks. In addition we recommend +consideration of the following points. + +BFD should be enabled if possible on all BGP sessions - both to the $[prodname] +nodes, and between racks in your core infrastructure - so that a break in connectivity +anywhere can be rapidly detected. The handling should be to initiate LLGR procedures if +possible, or else terminate the BGP session non-gracefully. + +LLGR should be enabled if possible on all BGP sessions - again, both to the +$[prodname] nodes, and between racks in your core infrastructure. Traditional BGP +graceful restart should not be used, because this will delay the cluster's response to a +break in one of the connectivity planes. + +### Install Kubernetes and $[prodname] + +Details here vary, depending on **when** your Kubernetes installer gives an opportunity +for you to define custom resources, but fundamentally what is needed here is to perform +the installation as usual, except that all of the Calico resources prepared above, except +the EarlyNetworkConfiguration, must be added into the datastore **before** the +$[nodecontainer] pods start running on any node. We can illustrate this by looking +at two examples: with OpenShift, and when adding Calico to an existing Kubernetes cluster. + +**OpenShift** + +With OpenShift, follow our documentation as +far as the option to provide additional configuration. +{/*TODO-XREFS-CC */} +Then use `kubectl create configmap ...`, as that documentation says, to combine the +prepared BGPPeer, BGPConfiguration and IPPool resources into a `calico-resources` +ConfigMap. Place the generated file in the manifests directory for the OpenShift install. + +Also place the "bgp-layout" ConfigMap file in the manifests directory. + +Now continue with the OpenShift install process, and it will take care of adding those +resources into the datastore as early as possible. + +**Adding to an existing Kubernetes cluster** + +Follow our documentation +{/*TODO-XREFS-CC */} +as +far as the option for installing any custom Calico resources. Then use `calicoctl`, as +that documentation says, to install the prepared BGPPeer, BGPConfiguration and IPPool +resources. + +Also use `kubectl` to install the "bgp-layout" ConfigMap. + +Now continue with the $[prodname] install process, and you should observe each node +establishing BGP sessions with its ToRs. + +### Verify the deployment + +If you examine traffic and connections within the cluster - for example, using `ss` or +`tcpdump` - you should see that all connections use loopback IP addresses or pod CIDR IPs +as their source and destination. For example: + +- The kubelet on each node connecting to the API server. + +- The API server's connection to its backing etcd database, and peer connections between + the etcd cluster members. + +- Pod connections that involve an SNAT or MASQUERADE in the data path, as can be the case + when connecting to a Service through a cluster IP or NodePort. At the point of the + SNAT or MASQUERADE, a loopback IP address should be used. + +- Direct connections between pod IPs on different nodes. + +The only connections using interface-specific addresses should be BGP. + +:::note + +If you plan to use [Egress Gateways](../egress/egress-gateway-on-prem.mdx) in your cluster, you must also adjust the $[nodecontainer] IP +auto-detection method to pick up the stable IP, for example using the `interface: lo` setting +(The default first-found setting skips over the lo interface). This can be configured via the +$[prodname] [Installation resource](../../reference/installation/api.mdx#nodeaddressautodetection). + +::: + +If you look at the Linux routing table on any cluster node, you should see ECMP routes +like this to the loopback address of every other node in other racks: + +``` +172.31.20.4/32 + nexthop via 172.31.11.250 dev eth0 + nexthop via 172.31.12.250 dev eth1 +``` + +and like this to the loopback address of every other node in the same rack: + +``` +172.31.10.4/32 + nexthop dev eth0 + nexthop dev eth1 +``` + +If you launch some pods in the cluster, you should see ECMP routes for the /26 IP blocks +for the nodes where those pods were scheduled, like this: + +``` +10.244.192.128/26 + nexthop via 172.31.11.250 dev eth0 + nexthop via 172.31.12.250 dev eth1 +``` + +If you do something to break the connectivity between racks of one of the planes, you +should see, within only a few seconds, that the affected routes change to have a single +path only, via the plane that is still unbroken; for example: + +``` +172.31.20.4/32 via 172.31.12.250 dev eth1` +10.244.192.128/26 via 172.31.12.250 dev eth1 +``` + +When the connectivity break is repaired, those routes should change to become ECMP again. + +## Additional resources + + - [L3 fabric designs](../../reference/architecture/design/l3-interconnect-fabric.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/index.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/index.mdx new file mode 100644 index 0000000000..c33e2bd4b8 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/index.mdx @@ -0,0 +1,11 @@ +--- +description: Networking configuration tasks for Calico Cloud connected clusters — BGP, overlay encapsulation, MTU, multiple pod networks, dual ToR, IPVS, NAT, and QoS controls. +hide_table_of_contents: true +--- + +# Configure Calico Cloud networking + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/local-subnet-l2-reachability.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/local-subnet-l2-reachability.mdx new file mode 100644 index 0000000000..083df0ac72 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/local-subnet-l2-reachability.mdx @@ -0,0 +1,178 @@ +--- +description: Make pods and LoadBalancer service VIPs that share the host subnet reachable from the local L2 network without BGP, using Calico Cloud's userspace ARP/NDP responder. +--- + +# L2 reachability for pods and services without BGP + +## Big picture + +Make pods and LoadBalancer service IPs reachable from the local layer 2 (L2) network without configuring BGP, by having $[prodname] answer ARP and NDP requests for those IPs directly. + +## Value + +When a pod or LoadBalancer VIP is assigned an IP from the same subnet as a node's physical interface, external hosts on that L2 segment try to reach it with ARP (IPv4) or Neighbor Solicitation (IPv6). By default nobody answers: the pod IP lives in a network namespace behind a veth, and a LoadBalancer VIP does not exist on any interface. + +Traditionally the only way to make these IPs reachable is to [advertise them over BGP](advertise-service-ips.mdx), which requires peering with your network infrastructure. In flat L2 environments — on-prem environment, edge deployments, `kind` clusters, or networks where you cannot configure the upstream router — BGP may not be an option. + +With `localSubnetL2Reachability` enabled, $[prodname] runs a userspace ARP/NDP responder on the relevant host interfaces and answers for local pod IPs and selected LoadBalancer VIPs that fall within the host subnet. No BGP, no overlay encapsulation, and no changes to your network infrastructure. + +## Concepts + +### Userspace ARP/NDP responder + +$[prodname] opens a raw socket on each host physical interface that has at least one pod IP or LoadBalancer VIP within its subnet, and replies to ARP/NDP requests for those IPs. The reply carries the node's MAC address, so the external host sends the workload's traffic to that node. From there $[prodname]'s normal data plane forwards it the rest of the way — directly to a local pod for a pod IP, or load-balanced to a service backend for a LoadBalancer VIP. The responder only answers ARP/NDP; it does not change how traffic is forwarded once it reaches the node. + +### One node answers per LoadBalancer VIP + +For a LoadBalancer VIP, exactly one node answers ARP. $[prodname] elects that node consistently across the cluster using a consistent-hash ring, so every node independently agrees on the owner without coordination. If the owning node's `Node` object is deleted, the VIP is reassigned to another node, which proactively updates external caches with a gratuitous ARP / unsolicited Neighbor Advertisement. + +### Only no-encapsulation IP pools are eligible + +$[prodname] answers only for IPs that belong to an IP pool with no encapsulation (`ipipMode` and `vxlanMode` both `Never`). Encapsulated pools are treated as private pod networks and are never advertised on the host L2 segment. + +## Before you begin... + +**Limitations** + +- **One node per LoadBalancer VIP.** There is no active-active high availability. If the owning node goes down, the VIP is unreachable until its `Node` object is removed and a new owner is elected. +- **Cloud provider ARP filtering.** Some cloud networks (for example, AWS VPC) filter ARP at the hypervisor. The responder will not work unless the VIP/pod IPs are assigned to the node's cloud interface. This is an infrastructure constraint outside $[prodname]'s control. +- **Heterogeneous subnets.** Node selection for LoadBalancer VIPs hashes over all nodes without checking whether a node has an interface on the VIP's subnet. This is only a concern when nodes do not share the same subnets. +- **Multiple host interfaces on the same L2 segment.** $[prodname] answers on *every* host interface whose subnet contains a proxied IP. If a node has two or more interfaces on the **same** L2 segment (the same broadcast domain) and a proxied IP falls in that shared subnet, each of those interfaces replies to ARP/NDP for the IP with its own MAC address, so external hosts see the IP resolve to several MACs and their neighbor caches flip between them. +- **Only `externalTrafficPolicy: Cluster` is supported for LoadBalancer services.** $[prodname] elects the node that answers for a VIP with a consistent hash over all nodes, independent of where the service's backends run. Under `externalTrafficPolicy: Local` the elected node may have no local backend, so traffic that reaches it is dropped. Leave `externalTrafficPolicy` at its default of `Cluster`. +- **kube-proxy in IPVS mode requires `strictARP`.** In IPVS mode, kube-proxy binds every LoadBalancer VIP to the `kube-ipvs0` dummy interface on *all* nodes. Because the Linux kernel answers ARP for any IP configured on the host by default, every node replies for the VIP — not just the one $[prodname] elected — so external hosts receive conflicting replies from several MAC addresses. Either run kube-proxy in `iptables` or `nftables` mode, or keep IPVS mode and set `strictARP: true` in kube-proxy's configuration (the `ipvs.strictARP` field of the `KubeProxyConfiguration`, which is stored as YAML inside the `kube-proxy` `ConfigMap`) so the kernel stops answering ARP for the dummy-interface VIPs and leaves it to $[prodname]'s responder. + +:::note + +The ARP/NDP responder runs inside Felix (`calico-node`). While a node's Felix is restarting it briefly does not answer ARP or NDP. Established connections keep working because the upstream router and switches still have the node's MAC cached for those IPs, but **a new connection to an IP that is not already cached upstream may fail until Felix finishes starting** and rebuilds its responder state (typically under a minute). + +To reduce the impact, increase the ARP/neighbor cache timeout (aging time) on your upstream router and switches so cached entries survive a Felix restart. + +::: + +## How to + +- [Enable local subnet L2 reachability](#enable-local-subnet-l2-reachability) +- [Create a dedicated IP pool in the host subnet](#create-a-dedicated-ip-pool-in-the-host-subnet) +- [Steer pods or services into the pool](#steer-pods-or-services-into-the-pool) +- [Verify reachability](#verify-reachability) + +### Enable local subnet L2 reachability + +Set `localSubnetL2Reachability` to `PodsAndLoadBalancers` on the default `FelixConfiguration`: + +```bash +kubectl patch felixconfiguration default --type='merge' \ + -p '{"spec": {"localSubnetL2Reachability": "PodsAndLoadBalancers"}}' +``` + +The feature is `Disabled` by default. This setting is read when the data plane starts, so changing it makes Felix (`calico-node`) **restart automatically** to pick up the new value — no manual restart is required. The feature takes effect once Felix finishes restarting. For the full field reference, see [Felix configuration resource](../../reference/resources/felixconfig.mdx). + +:::note + +Enabling the feature alone does nothing. $[prodname] only answers for IPs in a no-encapsulation IP pool that overlaps a host subnet, so you must also create that pool. + +::: + +**Optional: tune periodic re-announcement.** While the feature is enabled, Felix periodically re-announces every IP it proxies with a gratuitous ARP / unsolicited Neighbor Advertisement, keeping upstream neighbor caches and switch forwarding tables warm even when the set of proxied IPs has not changed. `localSubnetL2ReachabilityRefreshInterval` controls how often this happens (default `120s`). Lower it if your upstream devices age out ARP/NDP entries quickly, or set it to `0` to disable periodic re-announcement and keep only the one-shot announcement sent when an IP is first proxied: + +```bash +kubectl patch felixconfiguration default --type='merge' \ + -p '{"spec": {"localSubnetL2ReachabilityRefreshInterval": "30s"}}' +``` + +### Create a dedicated IP pool in the host subnet + +Create an IP pool whose CIDR lies inside your host interface's subnet, with no encapsulation. + +:::caution + +Reserve a sub-range of the host subnet for this pool that does **not** overlap DHCP scopes, static host assignments, or infrastructure addresses (routers, switches, management interfaces). For example, for a host subnet of `10.0.0.0/23`, use `10.0.0.2–10.0.0.255` for hosts and a separate `10.0.1.0/24` for the pool. + +::: + +**For pods:** + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: host-subnet-pod-pool +spec: + cidr: 10.0.1.0/24 + ipipMode: Never + vxlanMode: Never + natOutgoing: false + disabled: false + assignmentMode: Manual +``` + +`assignmentMode: Manual` keeps the pool opt-in: $[prodname] never assigns from it automatically, so only the workloads you explicitly steer into it get host-subnet IPs. Without this, $[prodname] could allocate IPs from the pool to arbitrary pods. + +**For LoadBalancer services**, add `allowedUses: [LoadBalancer]`: + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: host-subnet-lb-pool +spec: + cidr: 10.0.2.0/24 + ipipMode: Never + vxlanMode: Never + natOutgoing: false + disabled: false + assignmentMode: Manual + allowedUses: + - LoadBalancer +``` + +With `assignmentMode: Manual`, ordinary LoadBalancer services keep drawing VIPs from your existing automatic pools — you do not need to change the LoadBalancer kube-controller's `assignIPs` mode. Only services you annotate take a VIP from this host-subnet pool. + +Your cluster's default workload pool is unchanged, and workloads allocated from it behave exactly as before. For more on LoadBalancer IP pools, see [Configure LoadBalancer IP address management](../ipam/service-loadbalancer.mdx). + +### Steer pods or services into the pool + +Pods and services do not use the new pool automatically — direct the specific workloads you want reachable on the host L2 segment into it. + +**Pods:** annotate the pod or its namespace with the pool name: + +```yaml +cni.projectcalico.org/ipv4pools: '["host-subnet-pod-pool"]' +``` + +**LoadBalancer services:** annotate the `Service` with the pool name: + +```yaml +projectcalico.org/ipv4pools: '["host-subnet-lb-pool"]' +``` + +For IPv6, use `projectcalico.org/ipv6pools`. Services without this annotation continue to get VIPs from your automatic LoadBalancer pools as before. See [Configure LoadBalancer IP address management](../ipam/service-loadbalancer.mdx) for more options. + +### Verify reachability + +On the node, confirm ARP requests arrive and replies leave with the node's MAC: + +```bash +tcpdump -i arp +``` + +For IPv6 NDP solicitations and advertisements: + +```bash +tcpdump -i 'icmp6 and (icmp6.type == 135 or icmp6.type == 136)' +``` + +From an external host on the same L2 segment, confirm end-to-end resolution: + +```bash +arping -I +``` + +If there is no reply, check that the IP falls within both a host interface subnet and a no-encapsulation IP pool, that the feature is enabled, and that your network does not filter ARP (see Limitations). + +## Additional resources + +- [Advertise Kubernetes service IP addresses (over BGP)](advertise-service-ips.mdx) +- [Configure LoadBalancer IP address management](../ipam/service-loadbalancer.mdx) +- [Felix configuration resource](../../reference/resources/felixconfig.mdx) +- [IP pool resource](../../reference/resources/ippool.mdx) \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/mark-lb-node-for-maintenance.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/mark-lb-node-for-maintenance.mdx new file mode 100644 index 0000000000..f8a2256055 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/mark-lb-node-for-maintenance.mdx @@ -0,0 +1,61 @@ +--- +description: Mark a node in a Calico Cloud connected cluster for load balancer maintenance with an annotation so the eBPF data plane stops sending new service traffic to its pods. +--- + +# Mark a load balancer node for maintenance + +This guide shows you how to mark a node as being under maintenance. + +When a node is marked for maintenance, $[prodname] will avoid load balancing new service traffic +to pods on that node until the annotation is removed. Existing connections will continue to work +until they are finished. + + +## Limitations + +The following limitations apply to load balancer maintenance features: + + - Load balancer node maintenance features are only supported by the eBPF data plane. + + - Upstream routers might still send traffic to marked nodes, if routing is not configured to exclude + the marked nodes. In this case, the node under maintenance will load balance traffic to a backend + on another node. [Maglev Load Balancing](add-maglev-load-balancing.mdx) can be used to mitigate the negative effects of upstream routing + adjustments. + + +## Prerequisites + +* Your cluster uses the [eBPF data plane](../../operations/ebpf/enabling-ebpf.mdx). + + +## Exclude backends on a node from load balancer backend selection + +To exclude a node's backends from load balancer backend selection, annotate the Kubernetes node +resource with the following annotation: + +```yaml +lb.projectcalico.org/maintenance: "exclude-local-backends" +``` + +or from the terminal: + +```sh +kubectl annotate node 'lb.projectcalico.org/maintenance=exclude-local-backends' +``` + +To add the backends back into the selection pool, either remove the annotation, or set the value to "none". + + +## Additional notes + +### Failsafes + +It is not possible to remove all backends of a service from backend selection via the maintenance annotation. +In the event that the maintenance annotation would lead to an empty set of backends for a given service, no +backends will be excluded at all for that service. + + +## Additional resources +* [Enable the eBPF data plane](../../operations/ebpf/enabling-ebpf.mdx) +* [Maglev Load Balancing](add-maglev-load-balancing.mdx) +* [Prometheus Metrics](../../reference/component-resources/node/felix/prometheus.mdx#load-balancer-metrics) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/mtu.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/mtu.mdx new file mode 100644 index 0000000000..7d5303e3da --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/mtu.mdx @@ -0,0 +1,119 @@ +--- +description: Tune the Calico Cloud MTU on the Installation resource so pod traffic matches the underlay, accounting for VXLAN, IP-in-IP, and WireGuard overhead. +--- + +# Configure MTU to maximize network performance + +## Big picture + +Configure the maximum transmission unit (MTU) for your $[prodname] environment. + +## Value + +Optimize network performance for workloads by configuring the MTU in $[prodname] to best suit your underlying network. + +Increasing the MTU can improve performance, and decreasing the MTU can resolve packet loss and fragmentation problems when it is too high. + +## Concepts + +### MTU and $[prodname] defaults + +The maximum transmission unit (MTU) setting determines the largest packet size that can be transmitted through your network. MTU is configured on the veth attached to each workload, and tunnel devices (if you enable IP in IP, VXLAN, or WireGuard). + +In general, maximum performance is achieved by using the highest MTU value that does not cause fragmentation or dropped packets on the path. Maximum bandwidth increases and CPU consumption may drop for a given traffic rate. The improvement is often more significant when pod to pod traffic is being encapsulated (IP in IP, VXLAN, or WireGuard), and splitting and combining such traffic cannot be offloaded to your NICs. + +By default, $[prodname] will auto-detect the correct MTU for your cluster based on node configuration and enabled networking modes. This guide explains how you can override auto-detection +of MTU by providing an explicit value if needed. + +To ensure auto-detection of MTU works correctly, make sure that the correct encapsulation modes are set in your [felix configuration](../../reference/resources/felixconfig.mdx). Disable any unused encapsulations (`vxlanEnabled`, `ipipEnabled`, `wireguardEnabled` and `wireguardEnabledV6`) in your felix configuration to ensure that auto-detection can pick the optimal MTU for your cluster. + +## Before you begin... + +**Required** + +- Calico CNI + +For help on using IP in IP and/or VXLAN overlays, see [Configure overlay networking](vxlan-ipip.mdx). + +For help on using WireGuard encryption, see [Configure WireGuard encryption](../../compliance/encrypt-cluster-pod-traffic.mdx). + +## How to + +- [Determine MTU size](#determine-mtu-size) +- [Configure MTU](#configure-mtu) +- [View current tunnel MTU values](#view-current-tunnel-mtu-values) + +### Determine MTU size + +The following table lists common MTU sizes for $[prodname] environments. Because MTU is a global property of the network path between endpoints, you should set the MTU to the minimum MTU of any path that packets may take. + +**Common MTU sizes** + +| Network MTU | $[prodname] MTU | $[prodname] MTU with IP-in-IP (IPv4) | $[prodname] MTU with VXLAN (IPv4) | $[prodname] MTU with VXLAN (IPv6) | $[prodname] MTU with WireGuard (IPv4) | $[prodname] MTU with WireGuard (IPv6) | +| ---------------------- | --------------------- | ------------------------------------------ | --------------------------------------- | ------------------------------------------- | ---- | ---- | +| 1500 | 1500 | 1480 | 1450 | 1430 | 1440 | 1420 | +| 9000 | 9000 | 8980 | 8950 | 8930 | 8940 | 8920 | +| 1500 (AKS) | 1500 | 1480 | 1450 | 1430 | 1340 | 1320 | +| 1460 (GCE) | 1460 | 1440 | 1410 | 1390 | 1400 | 1380 | +| 9001 (AWS Jumbo) | 9001 | 8981 | 8951 | 8931 | 8941 | 8921 | +| 1450 (OpenStack VXLAN) | 1450 | 1430 | 1400 | 1380 | 1390 | 1370 | + +**Recommended MTU for overlay networking** + +The extra overlay header used in IP in IP, VXLAN and WireGuard protocols, reduces the minimum MTU by the size of the header. (IP in IP uses a 20-byte header, IPv4 VXLAN uses a 50-byte header, IPv6 VXLAN uses a 70-byte header, IPv4 WireGuard uses a 60-byte header and IPv6 WireGuard uses an 80-byte header). + +When using AKS, the underlying network has an [MTU of 1400](https://docs.microsoft.com/en-us/azure/virtual-network/virtual-network-tcpip-performance-tuning#azure-and-vm-mtu), even though the network interface will have an MTU of 1500. +WireGuard sets the Don't Fragment (DF) bit on its packets, and so the MTU for WireGuard on AKS needs to be set to 60 bytes below (or 80 bytes for IPv6) the 1400 MTU of the underlying network to avoid dropped packets. + +If you have a mix of WireGuard and either IP in IP or VXLAN in your cluster, you should configure the MTU to be the smallest of the values of each encap type. The reason for this is that only WireGuard encapsulation will be used between any nodes where both have WireGuard enabled, and IP in IP or VXLAN will then be used between any nodes where both do not have WireGuard enabled. This could be the case if, for example, you are in the process of installing WireGuard on your nodes. + +Therefore, we recommend the following: + +- If you use IPv4 WireGuard encryption anywhere in your pod network, configure MTU size as “physical network MTU size minus 60”. +- If you use IPv6 WireGuard encryption anywhere in your pod network, configure MTU size as “physical network MTU size minus 80”. +- If you don't use WireGuard, but use IPv4 VXLAN anywhere in your pod network, configure MTU size as “physical network MTU size minus 50”. +- If you don't use WireGuard, but use IPv6 VXLAN anywhere in your pod network, configure MTU size as “physical network MTU size minus 70”. +- If you don't use WireGuard, but use only IP in IP, configure MTU size as “physical network MTU size minus 20” +- Set the workload endpoint MTU and the tunnel MTUs to the same value (so all paths have the same MTU) + +**eBPF mode** + +Implementation of NodePorts uses VXLAN tunnel to hand off packets from one node to another, therefore VXLAN MTU setting +is used to set the MTUs of workloads (veths) and should be “physical network MTU size minus 50” (see above). + +**MTU for flannel networking** + +When using flannel for networking, the MTU for network interfaces should match the MTU of the flannel interface. + +- If using flannel with VXLAN, use the “$[prodname] MTU with VXLAN” column in the table above for common sizes. + +### Configure MTU + +:::note + +The updated MTU used by $[prodname] only applies to new workloads. + +::: + +For Operator installations, edit the $[prodname] operator `Installation` resource to set the `mtu` +field in the `calicoNetwork` section of the `spec`. For example: + +```bash +kubectl patch installation.operator.tigera.io default --type merge -p '{"spec":{"calicoNetwork":{"mtu":1440}}}' +``` + +Similarly, for OpenShift: + +```bash +oc patch installation.operator.tigera.io default --type merge -p '{"spec":{"calicoNetwork":{"mtu":1440}}}' +``` + +### View current tunnel MTU values + +To view the current tunnel size, use the following command: + +`ip link show` + +The IP in IP tunnel appears as tunlx (for example, tunl0), along with the MTU size. For example: + +![Tunnel MTU](/img/calico-enterprise/tunnel.png) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/multiple-networks.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/multiple-networks.mdx new file mode 100644 index 0000000000..94b28a08bd --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/multiple-networks.mdx @@ -0,0 +1,249 @@ +--- +description: Add extra Calico Cloud networks to each pod in a connected cluster with the Multus-CNI plugin, then control access with tiered network policy on every interface. +--- + +# Configure multiple Calico Cloud networks on a pod + +## Big picture + +Configure a Kubernetes cluster with multiple $[prodname] networks on each pod, and enforce security using $[prodname] tiered network policy. + +## Value + +By default, you can configure only one CNI (network and pod interface) in a cluster. But many deployments require multiple networks (for example, one that is faster or more secure) for sending different types of data. $[prodname] supports configuring additional $[prodname] networks and interfaces in your pods using the Multus-CNI plugin. You can then use $[prodname] tiered policy and other features to enforce security on all of your workload traffic. + +## Concepts + +### About the Multus-CNI plugin + +$[prodname] uses the [Multus-CNI plugin](https://github.com/intel/multus-cni/) to create multiple $[prodname] networks and multiple pod interfaces to access these networks. This extends the default network and pod interface that comes with the Calico CNI. + +You install Multus on a cluster, then simply enable Multus in the $[prodname] Installation resource. Using the Multus **NetworkAttachmentDefinition**, you define the new networks and reference them as an annotation in the pod resource. + +### Labels, workload endpoints, and policy + +When you set the `MultiInterfaceMode` field to `Multus` in the Installation resource, the following network and network interface labels are automatically added to new workload endpoints. + +- `projectcalico.org/network` +- `projectcalico.org/network-namespace` +- `projectcalico.org/network-interface` + +You can then create $[prodname] policies using these label selectors to target specific networks or network interfaces. + +### Limitations + +**Maximum additional networks per pod** + +You can define a maximum of nine additional $[prodname] networks on a pod. If you add a network that exceeds the limit for the pod, networking is not configured and the pod fails to start with an associated error. + +**$[prodname] features** + +Although the following $[prodname] features are supported for your default $[prodname] network, they are not supported at this time for additional networks/network interfaces using Multus: + +- Floating IPs +- Specific IPs +- Specifying IP pools on a per-namespace or per-pod basis +- Egress gateways + +## Before you begin... + +**Required** + +- Calico CNI + + :::note + + Verify that you are using the $[prodname] CNI. The CNI plugin used by Kubernetes for AKS, EKS, and GKE may be different, which means this feature will not work. + + ::: + +- [Install Multus 3.0+ on your Kubernetes cluster](https://github.com/intel/multus-cni/) + :::note + + Multus is installed on OpenShift 4.0+ clusters. + + ::: + +{/*//TODO-CC-XREFS- [Install and configure calicoctl](/operations/clis/calicoctl/index) or configure access to [the Calico Enterprise web console](/operations/cnx//access-the-manager/)*/} + +## How to + +1. [Configure cluster for multiple networks](#configure-cluster-for-multiple-networks) +1. [Create a new network](#create-a-new-network) +1. [Create a pod interface for the new network](#create-a-pod-interface-for-the-new-network) +1. [Configure the IP pool for the network](#configure-the-ip-pool-for-the-network) +1. [Enforce policy on the new network and pod interface](#enforce-policy-on-the-new-network-and-pod-interface) +1. [View workload endpoints](#view-workload-endpoints) + +### Configure cluster for multiple networks + +In the [Installation custom resource](../../reference/installation/api.mdx#caliconetworkspec), set the `MultiInterfaceMode` to **Multus**. + +### Create a new network + +Create a new network using the Multus **NetworkAttachmentDefinition**, and set the following required field to `"type":"calico"`. + +```yaml +apiVersion: 'k8s.cni.cncf.io/v1' +kind: NetworkAttachmentDefinition +metadata: + name: additional-calico-network +spec: + config: '{ + "cniVersion": "0.3.1", + "type": "calico", + "log_level": "info", + "datastore_type": "kubernetes", + "mtu": 1410, + "nodename_file_optional": false, + "ipam": { + "type": "calico-ipam", + "assign_ipv4" : "true", + "assign_ipv6" : "false" + }, + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "/etc/cni/net.d/calico-kubeconfig" + } + }' +``` + +### Create a pod interface for the new network + +Create a pod interface that specifies the new network using an annotation. + +In the following example, we create a pod with an additional pod interface named, `cali1`. The pod interface is attached to the network named, `additional-calico-network`, using the `k8s.v1.cni.cncf.io/networks` annotation. +Note that all networks in `k8s.v1.cni.cncf.io/networks` are assumed to be $[prodname] networks. + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: multus-test-pod-1 + namespace: default + annotations: + k8s.v1.cni.cncf.io/networks: additional-calico-network@cali1 +spec: + nodeSelector: + kubernetes.io/os: linux + containers: + - name: multus-test + command: ['/bin/sh', '-c', 'trap : TERM INT; sleep infinity & wait'] + image: alpine +``` + +### Configure the IP pool for the network + +Although not required, you may want to assign IPs from specific pools to specific network interfaces. If you are using the [$[prodname] IPAM plugin](../../reference/component-resources/configuration.mdx#specifying-ip-pools-on-a-per-namespace-or-per-pod-basis), specify the IP pools in the **NetworkAttachmentDefinition** custom resource. For example: + +``` + "ipam": { + "type": "calico-ipam", + "assign_ipv4" : "true", + "assign_ipv6" : "false" + "ipv4_pools": ["10.0.0.0/24", "20.0.0.0/16", "default-ipv4-ippool"], +}, +``` + +### Enforce policy on the new network and pod interface + +When MultiInterfaceMode is set to Multus, WorkloadEndpoints are created with these labels: + +- `projectcalico.org/network` +- `projectcalico.org/network-namespace` +- `projectcalico.org/network-interface` + +You can use these labels to enforce policies on specific interfaces and networks using policy label selectors. + +:::note + +Prior to $[prodname] 3.0, if you were using Kubernetes datastore (kdd mode), the workload endpoint field and name suffix were always **eth0**. In 3.0, the value for workload labels may not be what you expect. Before creating policies targeting WorkloadEndpoints using the new labels, you should verify label values using the commands in [View workload endpoints](#view-workload-endpoints). + +::: + +In this policy example, we use the selector field to target all WorkloadEndpoints with the network interface of, `cali1`. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: internal-access.allow-tcp-6379 + namespace: production +spec: + tier: internal-access + selector: projectcalico.org/network-interface == cali1 + types: + - Ingress + - Egress + ingress: + - action: Allow + metadata: + annotations: + from: frontend + to: database + protocol: TCP + source: + selector: role == 'frontend' + destination: + ports: + - 6379 + egress: + - action: Allow +``` + +### View workload endpoints + +**In the $[prodname] web console**, go to the **WorkloadEndpoint** page to see all of the WorkloadEndpoints, including the network labels are for targeting WorkloadEndpoints with policy. + +**Using the CLI...** + +To view all WorkloadEndpoints for pods (default and new), use the following command. + +``` +MULTI_INTERFACE_MODE=multus calicoctl get workloadendpoints -o wide +``` + +``` +NAME WORKLOAD NODE NETWORKS INTERFACE PROFILES NATS +test--bo--72vg--kadm--infra--0-k8s-multus--test--pod--1-eth0 multus-test-pod-1 bryan-bo-72vg-kadm-infra-0 192.168.53.129/32 calif887e436e8b kns.default,ksa.default.default +test--bo--72vg--kadm--infra--0-k8s-multus--test--pod--1-net1 multus-test-pod-1 bryan-bo-72vg-kadm-infra-0 192.168.53.140/32 calim17CD6INXIX kns.default,ksa.default.default +test--bo--72vg--kadm--infra--0-k8s-multus--test--pod--1-testiface multus-test-pod-1 bryan-bo-72vg-kadm-infra-0 192.168.53.142/32 calim27CD6INXIX kns.default,ksa.default.default +test--bo--72vg--kadm--infra--0-k8s-multus--test--pod--1-net3 multus-test-pod-1 bryan-bo-72vg-kadm-infra-0 192.168.52.143/32 calim37CD6INXIX kns.default,ksa.default.default +``` + +To view specific WorkloadEndpoints, use the following command. + +``` +MULTI_INTERFACE_MODE=multus calicoctl get workloadendpoint test--bz--72vg--kadm--infra--0-k8s-multus--test--pod--1-net1 -o yaml +``` + +```yaml +apiVersion: projectcalico.org/v3 +kind: WorkloadEndpoint +metadata: + creationTimestamp: '2020-05-04T22:23:05T' + labels: + projectcalico.org/namespace: default + projectcalico.org/network: calico + projectcalico.org/network-interface: net1 + projectcalico.org/network-namespace: default + projectcalico.org/orchestrator: k8s + projectcalico.org/serviceaccount: default + name: test--bz--72vg--kadm--infra--0-k8s-multus--test--pod--1-net1 + namespace: default + resourceVersion: '73572' + uid: b9bb7482-cdb8-48d4-9ae5-58322d48391a +spec: + endpoint: net1 + interfaceName: calim16CD6INXIX + ipNetworks: + - 192.168.52.141/32 + node: bryan-bo-72vg-kadm-infra-0 + orchestrator: k8s + pod: multus-test-pod-1 + profiles: + - kns.default + - ksa.default.default +``` diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/node-local-dns-cache.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/node-local-dns-cache.mdx new file mode 100644 index 0000000000..90d45b1a34 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/node-local-dns-cache.mdx @@ -0,0 +1,63 @@ +--- +description: Run NodeLocal DNSCache alongside Calico Cloud in a connected cluster and write the network policy that lets pod DNS traffic reach the per-node cache. +--- + +# Use NodeLocal DNSCache in your cluster + +## Big picture + +Set up NodeLocal DNSCache to improve DNS lookup latency. + +## Before you begin + +### Required + +Follow these [steps](https://kubernetes.io/docs/tasks/administer-cluster/nodelocaldns/) to enable NodeLocal DNSCache connectivity. + + +## Create a policy to allow traffic from NodeLocal DNSCache + +The following is a sample network policy that allows all incoming TCP traffic (including incoming traffic from +`node-local-dns` pods) on port 53 on `kube-dns`. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: default.local-dns-to-core-dns + namespace: kube-system +spec: + tier: default + selector: k8s-app == "kube-dns" + ingress: + - action: Allow + protocol: TCP + destination: + selector: k8s-app == "kube-dns" + ports: + - '53' + types: + - Ingress +``` + +To refine the sources permitted by this policy, take into account that NodeLocal DNSCache pods are host networked, +and make sure to allow traffic from the addresses of your hosts. +If you're using encapsulation, you will need to allow connectivity from the tunnel IPs. + +The Tigera Operator creates policy to allow Tigera components to connect to NodeLocal DNSCache when detected. +Felix accounts for the NodeLocal DNSCache in creating DNS Logs and enforcing DNS Policy. + +## Enable NodeLocal DNSCache on $[prodname] using the eBPF data plane + +If your $[prodname] installation uses the eBPF data plane, then you need to annotate the `kube-dns` service. +Without this, the `kube-proxy` replacement will improperly resolve traffic going to a specific service IP. + +Annotate the `kube-dns` service by running the following command: + +```bash +kubectl annotate service kube-dns -n kube-system projectcalico.org/natExcludeService=true +``` + +## Additional resources + +- [Blog: Optimizing Kubernetes DNS with NodeLocal DNSCache and Calico eBPF: A Practitioner’s Guide](https://www.tigera.io/blog/optimizing-kubernetes-dns-with-nodelocal-dnscache-and-calico-ebpf-a-practitioners-guide/) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/pod-mac-address.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/pod-mac-address.mdx new file mode 100644 index 0000000000..2e92b5544f --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/pod-mac-address.mdx @@ -0,0 +1,35 @@ +--- +description: Pin a chosen MAC address on a Kubernetes pod interface in a Calico Cloud connected cluster with the CNI plugin, for cases such as MAC-bound software licenses. +--- + +# Use a specific MAC address for a pod + +## Big picture + +Choose the MAC address for a pod instead of allowing the operating system to assign one. + +## Value + +Some applications bind software licenses to networking interface MAC addresses. + +## Concepts + +### Container MAC address + +The MAC address configured by the annotation described here will be visible from within the container on the eth0 interface. Since it is isolated to the container it will not collide with any other MAC addresses assigned to other pods on the same node. + +## Before you begin... + +Your cluster must be using Calico CNI to use this feature. + +[Configuring the Calico CNI Plugins](../../reference/component-resources/configuration.mdx) + +## How to + +Annotate the pod with cni.projectcalico.org/hwAddr set to the desired MAC address. For example: + +``` + "cni.projectcalico.org/hwAddr": "1c:0c:0a:c0:ff:ee" +``` + +The annotation must be present when the pod is created; adding it later has no effect. diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/qos-controls.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/qos-controls.mdx new file mode 100644 index 0000000000..83bf8cb264 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/qos-controls.mdx @@ -0,0 +1,148 @@ +--- +description: Apply Calico Cloud QoS controls to cap pod ingress and egress bandwidth, packet rate, and connection counts in a connected cluster, plus DiffServ marking on egress. +--- + +# Configure QoS Controls + +## Big picture + +Configure QoS (Quality of Service) Controls to limit ingress and/or egress bandwidth, packet rate and number of connections used by $[prodname] workloads in order to prevent them from overusing or overloading network resources. +Additionally, it allows configuring Differentiated Services (DiffServ) on egress traffic of $[prodname] workloads and hosts. + +## Value + +With QoS controls, $[prodname] enforces network resource limits (bandwidth, packet rate, connections) for Kubernetes pods and OpenStack VMs, ensuring fair resource allocation and preventing performance degradation for other workloads. +$[prodname] can also apply DiffServ on egress traffic to allow upstream devices to prioritise forwarding accordingly. + +## Prerequisites + +* If you use the eBPF data plane, your Linux nodes must be running kernel version 6.6 or higher in order to configure bandwidth QoS controls. + +## Concepts + +### Quality of Service Controls + +**Quality of Service** (**QoS**), in the field of computer networking, refers to traffic prioritization and resource reservation control mechanisms. It may either mean guaranteeing minimum levels of networking resources, or limiting the maximum levels of usage of networking resources in order to prevent a small number of users from excessive resource consumption, stopping others from using their fair share. For practical purposes, the latter will be considered as $[prodname] implements QoS Controls by limiting maximum levels of network resources usage. + +$[prodname] implements three types of QoS Controls, which can be configured using annotations on the Kubernetes pods for either ingress (traffic coming into the pod), egress (traffic going out of the pod), or both: +1. Bandwidth: limiting the bit rate of traffic into/out of the pod +2. Packet rate: limiting the number of packets per second that can be sent or received by the pod +3. Number of established connections: limiting the absolute number of connections that the pod may initiate or accept +4. Differentiated Services (DiffServ): setting Differentiated Services Code Point (DSCP) on egress traffic leaving clusters or toward hosts + +### Annotations + +On Kubernetes, $[prodname] uses [annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) on pods for the configuration of QoS controls. + +The values of the QoS controls annotations may contain suffixes, such as `k`, `M`, `G`, `T`, `P`, or `E` to represent large numbers. Binary suffixes are also supported, for example `1Ki` represents "1024". + +For example, to limit egress bandwidth to 100Mbps, with a maximum burst of 200Mb: + +```yaml noValidation +apiVersion: apps/v1 +kind: Deployment +metadata: + name: my-deployment + labels: + app: my-app +spec: + replicas: 3 + template: + metadata: + annotations: + qos.projectcalico.org/egressBandwidth: "100M" + qos.projectcalico.org/egressBurst: "200M" + spec: + (...) +``` + +## How to + +- [Configure bandwidth limits](#configure-bandwidth-limits) +- [Configure packet rate limits](#configure-packet-rate-limits) +- [Configure established connection limits](#configure-established-connection-limits) +- [Configure DiffServ](#configure-diffserv) + +### Configure bandwidth limits + +Use the annotations below to configure bandwidth and burst limits for traffic going into or coming out of the pod. Limiting the bandwidth enforces that, on average, that will be the maximum bit rate used. The burst size is the amount of data that is available instantaneously for transmission. Burst configuration is optional, if burst is not explicitly configured with an annotation, a default value of `4Gib` is used. If you configure a peak rate limit, burst traffic will not go over that limit. Otherwise, there are no limits to burst traffic. + +| Annotation | Description | +|---------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `qos.projectcalico.org/ingressBandwidth` | Specifies the ingress bandwidth limit (in bits per second).
Min value: `1k`
Max value: `1P` | +| `qos.projectcalico.org/ingressBurst` | Specifies the ingress burst size (in bits).
Optional, defaults to `4Gi`.
Min value: `1k`
Max value: `34359738360` | +| `qos.projectcalico.org/ingressPeakrate` | Specifies the ingress peak rate limit, i.e. the bandwidth limit during bursts (in bits per second).
Optional, defaults to unlimited.
Must be higher than the bandwidth limit.
Min value: `1.01k`
Max value: `1.01P` | +| `qos.projectcalico.org/ingressMinburst` | Specifies the ingress minburst, i.e. the burst size for peak rate (in **bytes**).
Optional, defaults to the MTU value.
Min value: `1k`
Max value: `100M` | +| `qos.projectcalico.org/egressBandwidth` | Specifies the egress bandwidth limit (in bits per second).
Min value: `1k`
Max value: `1P` | +| `qos.projectcalico.org/egressBurst` | Specifies the egress burst size (in bits).
Optional, defaults to `4Gi`.
Min value: `1k`
Max value: `34359738360` | +| `qos.projectcalico.org/egressPeakrate` | Specifies the egress peak rate limit, i.e. the bandwidth limit during bursts (in bits per second).
Optional, defaults to unlimited.
Must be higher than the bandwidth limit.
Min value: `1.01k`
Max value: `1.01P` | +| `qos.projectcalico.org/egressMinburst` | Specifies the egress minburst, i.e. the burst size for peak rate (in **bytes**).
Optional, defaults to the MTU value.
Min value: `1k`
Max value: `100M` | + + +If the [Kubernetes bandwidth plugin](https://kubernetes.io/docs/concepts/extend-kubernetes/compute-storage-net/network-plugins/) annotations `kubernetes.io/ingress-bandwidth` or `kubernetes.io/egress-bandwidth` are configured and the `qos.projectcalico.org` annotations are not, they are honored and $[prodname] will configure bandwidth limits based on them. The $[prodname] specific annotations take precedence, so if both are configured, the Kubernetes annotation will be ignored. This makes upgrading from a $[prodname] version that uses the bandwidth plugin to a version with native bandwidth limiting seamless, as the behavior will remain consistent across versions (with $[prodname] replacing the functionality previously provided by the Kubernetes bandwidth plugin). + +One essential difference between the bandwidth plugin and $[prodname]'s implementations is that with $[prodname], the bandwidth limits take effect immediately, whereas with the bandwidth plugin the pods need to be recreated for the limits to take effect. + +| Annotation | Description | +|---------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `kubernetes.io/ingress-bandwidth` | Kubernetes bandwidth plugin annotation for the ingress bandwidth limit (in bits per second).
Ignored if `qos.projectcalico.org/ingressBandwidth` is present.
Min value: `1k`
Max value: `1P` | +| `kubernetes.io/egress-bandwidth` | Kubernetes bandwidth plugin annotation for the egress bandwidth limit (in bits per second).
Ignored if `qos.projectcalico.org/egressBandwidth` is present.
Min value: `1k`
Max value: `1P` | + +### Configure packet rate limits + +Use the annotations below to configure packet rate limits for traffic going into or coming out of the pod. Limiting the packet rate enforces that, on average, that will be the maximum rate that packets will be sent and/or received by the pod. The packet burst size is the number of packets that are available instantaneously for transmission. Packet burst configuration is optional. If packet burst is not explicitly configured with an annotation, a default value of `5` packets is used. + +| Annotation | Description | +|---------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `qos.projectcalico.org/ingressPacketRate` | Specifies the maximum rate of ingress packets (in packets per second).
Min value: `1`
Max value: `10k` | +| `qos.projectcalico.org/ingressPacketBurst` | Specifies the ingress packet burst size (in number of packets).
Optional, defaults to `5`.
Min value: `1`
Max value: `10k` | +| `qos.projectcalico.org/egressPacketRate` | Specifies the maximum rate of egress packets (in packets per second).
Min value: `1`
Max value: `10k` | +| `qos.projectcalico.org/egressPacketBurst` | Specifies the egress packet burst size (in number of packets).
Optional, defaults to `5`.
Min value: `1`
Max value: `10k` | + +### Configure established connection limits + +Use the annotations below to configure limits for the number of established connections into or out of a pod. Limiting the number of connections enforces that the pod cannot accept or initiate more connections that the specified limits. + +| Annotation | Description | +|---------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `qos.projectcalico.org/ingressMaxConnections` | Specifies the maximum number of ingress connections.
Min value: `1`
Max value: `4294967295` | +| `qos.projectcalico.org/egressMaxConnections` | Specifies the maximum number of egress connections.
Min value: `1`
Max value: `4294967295` | + +### Configure DiffServ + +Use the annotation below to set DSCP on egress traffic toward an endpoint outside cluster or another host in cluster. Unlike, other QoS controls, this annotation can also be set on a [host endpoint](../../reference/host-endpoints). +For example, to set DSCP AF11 on a host endpoint: + +```yaml noValidation +apiVersion: projectcalico.org/v3 +kind: HostEndpoint +metadata: + name: my-hep + annotations: + qos.projectcalico.org/dscp: "AF11" + labels: + app: my-app +spec: + (...) +``` + +| Annotation | Description | +|---------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `qos.projectcalico.org/dscp` | Specifies the DSCP value of egress traffic toward an endpoint outside cluster or another host in cluster.
Min value: `0`
Max value: `63`
String values: DF, EF, AF11, AF12, AF13, AF21, AF22, AF23, AF31, AF32, AF33, AF41, AF42, AF43, CS0, CS1, CS2, CS3, CS4, CS5, CS6 | + +## Troubleshooting + +### Bandwidth is not enforced on eBPF + +If you use the eBPF data plane and bandwidth limits aren't being enforced, check the calico-node logs to see if `tcx` is supported by the operating system. +`tcx` is present in the Linux kernel version 6.6 and later, and it is required for bandwidth limits to work with the eBPF data plane. + +The following example shows logs that indicate an unsupported kernel version: + +``` +$ kubectl logs -n calico-system ds/calico-node | grep tcx +Found 4 pods, using pod/calico-node-j9z5d +Defaulted container "calico-node" out of: calico-node, flexvol-driver (init), ebpf-bootstrap (init), install-cni (init) +bird: libbpf: prog 'cali_tcx_test': failed to attach to tcx: Invalid argument +2025-10-15 16:31:48.522 [INFO][81] felix/bpf_ep_mgr.go 595: tcx is not supported. Falling back to tc +``` diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/secure-bgp.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/secure-bgp.mdx new file mode 100644 index 0000000000..93c0604801 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/secure-bgp.mdx @@ -0,0 +1,185 @@ +--- +description: Configure BGP session passwords on Calico Cloud BGP peers on a connected cluster to block attackers from injecting false routing information. +--- + +# Secure BGP sessions + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## Big picture + +Use BGP passwords to prevent attackers from injecting false routing information. + +## Value + +Setting a password on a BGP peering between BGP speakers means that a peering will only +work when both ends of the peering have the same password. This provides a layer of defense +against an attacker impersonating an external BGP peer or a workload in the cluster, for +example to inject malicious routing information into the cluster. + +## Concepts + +### Password protection on BGP sessions + +Password protection is a [standardized](https://tools.ietf.org/html/rfc5925) optional +feature of BGP sessions. The effect is that the two peers at either end of a BGP session +can only communicate, and exchange routing information, if they are both configured with +the same password. + +Please note that password use does not cause the data exchange to be _encrypted_. It +remains relatively easy to _eavesdrop_ on the data exchange, but not to _inject_ false +information. + +### Using Kubernetes secrets to store passwords + +In Kubernetes, the Secret resource is designed for holding sensitive information, +including passwords. Therefore, for this $[prodname] feature, we use Secrets to +store BGP passwords. + +## How to + +To use a password on a BGP peering: + +1. Create (or update) a Kubernetes secret in the namespace where $[noderunning] is + running, so that it has a key whose value is the desired password. Note the secret + name and the key name. + + :::note + + BGP passwords must be 80 characters or fewer. If a + password longer than that is configured, the BGP sessions with + that password will fail to be established. + + ::: + +1. Ensure that $[noderunning] has RBAC permissions to access that secret. + +1. Specify the secret and key name on the relevant BGPPeer resource. + +### Create or update Kubernetes secret + +For example: + +``` +kubectl create -f - < + + +When [configuring a BGP peer](../../networking/configuring/bgp.mdx), +include the secret and key name in the specification of the BGPPeer resource, like this: + +```yaml +apiVersion: projectcalico.org/v3 +kind: BGPPeer +metadata: + name: bgppeer-global-3040 +spec: + peerIP: 192.20.30.40 + asNumber: 64567 + password: + secretKeyRef: + name: bgp-secrets + key: rr-password +``` + + + + +Include the secret in the default [BGP configuration](../../reference/resources/bgpconfig.mdx) +similar to the following: + +```yaml +kind: BGPConfiguration +apiVersion: projectcalico.org/v3 +metadata: + name: default +spec: + logSeverityScreen: Info + nodeToNodeMeshEnabled: true + nodeMeshPassword: + secretKeyRef: + name: bgp-secrets + key: rr-password +``` + +:::note + +Node to node mesh must be enabled to set node to node mesh +BGP password. + +::: + + + + + +## Additional resources + +For more detail about the BGPPeer resource, see +[BGPPeer](../../reference/resources/bgppeer.mdx). + +For more on configuring BGP peers, see [configuring BGP peers](../../networking/configuring/bgp.mdx) +. diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/vxlan-ipip.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/vxlan-ipip.mdx new file mode 100644 index 0000000000..bcee245f11 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/vxlan-ipip.mdx @@ -0,0 +1,148 @@ +--- +description: Choose VXLAN or IP-in-IP overlay encapsulation in a Calico Cloud connected cluster so pod traffic crosses underlay networks that don't route pod CIDRs natively. +--- + +# Overlay networking + +## Big picture + +Enable inter workload communication across networks that are not aware of workload IPs. + +## Value + +In general, we recommend running Calico without network overlay/encapsulation. This gives you the highest performance and simplest network; the packet that leaves your workload is the packet that goes on the wire. + +However, selectively using overlays/encapsulation can be useful when running on top of an underlying network that cannot easily be made aware of workload IPs. A common example is if you are using Calico networking in AWS across multiple VPCs/subnets. In this case, Calico can selectively encapsulate only the traffic that is routed between the VPCs/subnets, and run without encapsulation within each VPC/subnet. You might also decide to run your entire Calico network with encapsulation as an overlay network -- as a quick way to get started without setting up BGP peering or other routing information in your underlying network. + +## Concepts + +### Routing workload IP addresses + +Networks become aware of workload IP addresses through layer 3 routing techniques like static routes or BGP route distribution, or layer 2 address learning. As such, they can route unencapsulated traffic to the right host for the endpoint that is the ultimate destination. However, not all networks are able to route workload IP addresses. For example, public cloud environments where you don’t own the hardware, AWS across VPC subnet boundaries, and other scenarios where you cannot peer Calico over BGP to the underlay, or easily configure static routes. This is why Calico supports encapsulation, so you can send traffic between workloads without requiring the underlying network to be aware of workload IP addresses. + +### Encapsulation types + +Calico supports two types of encapsulation: VXLAN and IP in IP. VXLAN is supported in some environments where IP in IP is not (for example, Azure). VXLAN has a slightly higher per-packet overhead because the header is larger, but unless you are running very network intensive workloads the difference is not something you would typically notice. The other small difference between the two types of encapsulation is that Calico's VXLAN implementation does not use BGP, whereas Calico's IP in IP implementation uses BGP between Calico nodes. + +### Cross-subnet + +Encapsulation of workload traffic is typically required only when traffic crosses a router that is unable to route workload IP addresses on its own. Calico can perform encapsulation on: all traffic, no traffic, or only on traffic that crosses a subnet boundary. + +## Before you begin + +**Required** + +- Calico CNI + +**Not supported** + +- Calico for OpenStack (i.e. when Calico is used as the Neutron plugin) + +**Limitations** + +- IP in IP supports only IPv4 addresses +- VXLAN in IPv6 is only supported for kernel versions ≥ 4.19.1 or redhat kernel version ≥ 4.18.0 + +## How to + +- [Configure default IP pools at install time](#configure-default-ip-pools-at-install-time) +- [Configure IP in IP encapsulation for only cross-subnet traffic](#configure-ip-in-ip-encapsulation-for-only-cross-subnet-traffic) +- [Configure IP in IP encapsulation for all inter workload traffic](#configure-ip-in-ip-encapsulation-for-all-inter-workload-traffic) +- [Configure VXLAN encapsulation for only cross-subnet traffic](#configure-vxlan-encapsulation-for-only-cross-subnet-traffic) +- [Configure VXLAN encapsulation for all inter workload traffic](#configure-vxlan-encapsulation-for-all-inter-workload-traffic) + +### IPv4/6 address support + +IP in IP supports only IPv4 addresses. + +### Best practice + +Calico has an option to selectively encapsulate only traffic that crosses subnet boundaries. We recommend using the **cross-subnet** option with IP in IP or VXLAN to minimize encapsulation overhead. Cross-subnet mode provides better performance in AWS multi-AZ deployments, Azure VNets, and on networks where routers are used to connect pools of nodes with L2 connectivity. + +Be aware that switching encapsulation modes can cause disruption to in-progress connections. Plan accordingly. + +### Configure default IP pools at install time + +Default IP pools are configured at install-time automatically by Calico. + +For operator managed clusters, you can configure encapsulation in the IP pools section of the default Installation. For example, the following installation snippet will enable VXLAN across subnets. + +```yaml +kind: Installation +apiVersion: operator.tigera.io/v1 +metadata: + name: default +spec: + calicoNetwork: + ipPools: + - cidr: 192.168.0.0/16 + encapsulation: VXLANCrossSubnet +``` + +### Configure IP in IP encapsulation for only cross-subnet traffic + +IP in IP encapsulation can be performed selectively, and only for traffic crossing subnet boundaries. + +To enable this feature, set `ipipMode` to `CrossSubnet`. + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: ippool-ipip-cross-subnet-1 +spec: + cidr: 192.168.0.0/16 + ipipMode: CrossSubnet + natOutgoing: true +``` + +### Configure IP in IP encapsulation for all inter workload traffic + +With `ipipMode` set to `Always`, Calico routes traffic using IP in IP for all traffic originating from a Calico enabled-host, to all Calico networked containers and VMs within the IP pool. + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: ippool-ipip-1 +spec: + cidr: 192.168.0.0/16 + ipipMode: Always + natOutgoing: true +``` + +### Configure VXLAN encapsulation for only cross subnet traffic + +VXLAN encapsulation can be performed selectively, and only for traffic crossing subnet boundaries. + +To enable this feature, set `vxlanMode` to `CrossSubnet`. + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: ippool-vxlan-cross-subnet-1 +spec: + cidr: 192.168.0.0/16 + vxlanMode: CrossSubnet + natOutgoing: true +``` + +### Configure VXLAN encapsulation for all inter workload traffic + +With `vxlanMode` set to `Always`, Calico routes traffic using VXLAN for all traffic originating from a Calico enabled host, to all Calico networked containers and VMs within the IP pool. + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: ippool-vxlan-1 +spec: + cidr: 192.168.0.0/16 + vxlanMode: Always + natOutgoing: true +``` + +## Additional resources + +For details on IP pool resource options, see [IP pool](../../reference/resources/ippool.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/networking/configuring/workloads-outside-cluster.mdx b/calico-cloud_versioned_docs/version-24-1/networking/configuring/workloads-outside-cluster.mdx new file mode 100644 index 0000000000..3f864337f3 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/configuring/workloads-outside-cluster.mdx @@ -0,0 +1,70 @@ +--- +description: Configure NAT outgoing on Calico Cloud IP pools in a connected cluster so pod traffic destined outside the cluster is source-NATed to the node IP. +--- + +# Configure outgoing NAT + +## Big picture + +Configure $[prodname] networking to perform outbound NAT for connections from pods to outside of the cluster. $[prodname] optionally source NATs the pod IP to the node IP. + +## Value + +The $[prodname] NAT outbound connection option is flexible; it can be enabled, disabled, and applied to $[prodname] IP pools with public IPs, private IPs, or a specific range of IP addresses. This article describes some use cases for enabling and disabling outgoing NAT. + +## Concepts + +### $[prodname] IP pools and NAT + +When a pod with an IP address in the pool initiates a network connection to an IP address to outside of $[prodname]’s IP pools, the outgoing packets will have their source IP address changed from the pod IP address to the node IP address using SNAT (Source Network Address Translation). Any return packets on the connection automatically get this change reversed before being passed back to the pod. + +### Enable NAT: for pods with IP addresses that are not routable beyond the cluster + +A common use case for enabling NAT outgoing, is to allow pods in an overlay network to connect to IP addresses outside of the overlay, or pods with private IP addresses to connect to public IP addresses outside the cluster/the internet (subject to network policy allowing the connection, of course). When NAT is enabled, traffic is NATed from pods in that pool to any destination outside of all other $[prodname] IP pools. + +### Disable NAT: For on-premises deployments using physical infrastructure + +If you choose to implement $[prodname] networking with [BGP peered with your physical network infrastructure](bgp.mdx), you can use your own infrastructure to NAT traffic from pods to the internet. In this case, you should disable the $[prodname] `natOutgoing` option. For example, if you want your pods to have public internet IPs, you should: + +- Configure $[prodname] to peer with your physical network infrastructure +- Create an IP pool with public IP addresses for those pods that are routed to your network with NAT disabled (`natOutgoing: false`) +- Verify that other network equipment does not NAT the pod traffic + +## Before you begin + +**Required** + +- Calico CNI + +## How to + +- [Create an IP pool with NAT outgoing enabled](#create-an-ip-pool-with-nat-outgoing-enabled) +- [Use additional IP pools to specify addresses that can be reached without NAT](#use-additional-ip-pools-to-specify-addresses-that-can-be-reached-without-nat) + +### Create an IP pool with NAT outgoing enabled + +In the following example, we create a $[prodname] IPPool with natOutgoing enabled. Outbound NAT is performed locally on the node where each workload in the pool is hosted. + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: default-ipv4-ippool +spec: + cidr: 192.168.0.0/16 + natOutgoing: true +``` + +### Use additional IP pools to specify addresses that can be reached without NAT + +Because $[prodname] performs outgoing NAT only when connecting to an IP address that is not in a $[prodname] IPPool, you can create additional IPPools that are not used for pod IP addresses, but prevent NAT to certain CIDR blocks. This is useful if you want nodes to NAT traffic to the internet, but not to IPs in certain internal ranges. For example, if you did not want to NAT traffic from pods to 10.0.0.0/8, you could create the following pool. You must ensure that the network between the cluster and 10.0.0.0/8 can route pod IPs. + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: no-nat-10.0.0.0-8 +spec: + cidr: 10.0.0.0/8 + disabled: true +``` \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/networking/egress/egress-gateway-aws.mdx b/calico-cloud_versioned_docs/version-24-1/networking/egress/egress-gateway-aws.mdx new file mode 100644 index 0000000000..a1d10dd4d7 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/egress/egress-gateway-aws.mdx @@ -0,0 +1,1174 @@ +--- +description: Route selected application traffic out of a Calico Cloud connected cluster through egress gateways with native AWS VPC subnet IPs. +--- + +# Configure egress gateways, AWS + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## Big picture + +Control the source IP address seen by external services/appliances by routing the traffic from certain pods +through egress gateways. Use native VPC subnet IP addresses for the egress gateways so that the IPs are valid in the AWS fabric. + +## Value + +Controlling the source IP seen when traffic leaves the cluster allows groups of pods to be identified +by external firewalls, appliances and services (even as the groups are scaled up/down or pods restarted). +$[prodname] controls the source IP by directing traffic through one or more "egress gateway" pods, which +change the source IP of the traffic to their own IP. The egress gateways used can be chosen at the pod or namespace +scope allowing for flexibility in how the cluster is seen from outside. + +In AWS, egress gateway source IP addresses are chosen from an IP pool backed by a [VPC subnet](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Subnets.html) +using $[prodname] IPAM. $[prodname] IPAM allows the IP addresses to be precisely controlled, this allows +for static configuration of external appliances. Using an IP pool backed by a VPC subnet allows $[prodname] to +configure the AWS fabric to route traffic to and from the egress gateway using its own IP address. + +## Concepts + +### CIDR notation + +This article assumes that you are familiar with network masks and CIDR notation. + +- CIDR notation is defined in [RFC4632](https://datatracker.ietf.org/doc/html/rfc4632). +- The [Wikipedia article on CIDR notation](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing#CIDR_notation) + provides a good reference. + +### AWS-backed IP pools + +$[prodname] supports IP pools that are backed by the AWS fabric. Workloads that use an IP address from an +AWS-backed pool can communicate on the AWS network using their own IP address and AWS will route their traffic +to/from their host without changing the IP address. + +Pods that use an IP address from an AWS-backed pool may also be [assigned an AWS Elastic IP via a pod annotation](#add-aws-elastic-ips-to-the-egress-gateway-deployment) +. Elastic IPs used in this +way have the normal AWS semantics: when accessing resources inside the AWS network, the workload's private IP +(from the IP pool) is used. When accessing resources outside the AWS network, AWS translates the workload's IP to +the Elastic IP. Elastic IPs also allow for incoming requests from outside the AWS fabric, direct to the workload. + +In overview, the AWS-backed IP Pools feature works as follows: + +- An IP pool is created with its `awsSubnetID` field set to the ID of a VPC subnet. This "AWS-backed" IP pool's + CIDR must be contained within the VPC subnet's CIDR. + + :::caution + + You must ensure that the CIDR(s) used for AWS-backed IP pool(s) are reserved in the AWS fabric. + For example, by creating a dedicated VPC subnet for $[prodname]. If the CIDR is not reserved; both + $[prodname] and AWS may try to assign the same IP address, resulting in a conflict. + + ::: + +- Since they are a limited resource, $[prodname] IPAM does not use AWS-backed pools by default. To request an + AWS-backed IP address, a pod must have a resource request: + + ```yaml noValidation + spec: + containers: + - ... + resources: + requests: + projectcalico.org/aws-secondary-ipv4: 1 + limits: + projectcalico.org/aws-secondary-ipv4: 1 + ``` + + $[prodname] manages the `projectcalico.org/aws-secondary-ipv4` capacity on the Kubernetes Node resource, + ensuring that Kubernetes will not try to schedule too many AWS-backed workloads to the same node. Only AWS-backed + pods are limited in this way; there is no limit on the number of non-AWS-backed pods. + +- When the CNI plugin spots such a resource request, it will choose an IP address from an AWS-backed pool. Only + pools with VPC subnets in the availability zone of the host are considered. + +- When Felix, $[prodname]'s per-host agent spots a local workload with an AWS-backed address it tries to ensure + that the IP address of the workload is assigned to the host in the AWS fabric. If need be, it will create a + new [secondary ENI](#secondary-elastic-network-interfaces-enis) device and attach it to the host to house the IP address. + Felix supports two modes for assigning secondary ENIs: **ENI-per-workload** mode (added in v3.13) and + **Secondary-IP-per-workload** mode. These modes are described [below](#secondary-elastic-network-interfaces-enis). + +- If the pod has one or more AWS Elastic IPs listed in the `cni.projectcalico.org/awsElasticIPs` pod annotation, + Felix will try to ensure that _one_ of the Elastic IPs is assigned to the pod's private IP address in the AWS fabric. + (Specifying multiple Elastic IPs is useful for multi-pod deployments; ensuring that each pod in the deployment + gets one of the IPs.) + +### Egress gateway + +An egress gateway acts as a transit pod for the outbound application traffic that is configured to +use it. As traffic leaving the cluster passes through the egress gateway, its source IP is changed +to that of the egress gateway pod, and the traffic is then forwarded on. + +### Source IP + +When an outbound application flow leaves the cluster, its IP packets will have a source IP. +This begins as the pod IP of the pod that originated the flow, then: + +- _If no egress gateway is configured_ and the pod IP came from an [IP pool](../../reference/resources/ippool.mdx) + with `natOutgoing: true`, the node hosting the pod will change the source IP to its own as the + traffic leaves the host. This allows the pod to communicate with external service even though the + external network is unaware of the pod's IP. + +- _If the pod is configured with an egress gateway_, the traffic is first forwarded to the egress gateway, which + changes the source IP to its own and then sends the traffic on. To function correctly, egress gateways + should have IPs from an IP pool with `natOutgoing: false`, meaning their host forwards the packet onto + the network without changing the source IP again. Since the egress gateway's IP is visible to + the underlying network fabric, the fabric must be configured to know about the egress gateway's + IP and to send response traffic back to the same host. + +### AWS VPCs and subnets + +An AWS VPC is a virtual network that is, by default, logically isolated from other VPCs. Each VPC has one or more +(often large) CIDR blocks associated with it (for example `10.0.0.0/16`). In general, VPC CIDRs may overlap, but only +if the VPCs remain isolated. AWS allows VPCs to be peered with each other through VPC Peerings. VPCs can only be +peered if _none_ of their associated CIDRs overlap. + +Each VPC has one or more VPC subnets associated with it, each subnet owns a non-overlapping part of one of the +VPC's CIDR blocks. Each subnet is associated with a particular availability zone. Instances in one availability +zone can only use IP addresses from subnets in that zone. Unfortunately, this adds some complexity to managing +egress gateways IP addresses: much of the configuration must be repeated per-AZ. + +### AWS VPC and DirectConnect peerings + +AWS [VPC Peerings](https://docs.aws.amazon.com/vpc/latest/peering/vpc-peering-basics.html) allow multiple VPCs to be +connected together. Similarly, [DirectConnect](https://docs.aws.amazon.com/directconnect/latest/UserGuide/Welcome.html) +allows external datacenters to be connected to an AWS VPC. Peered VPCs and datacenters communicate using private IPs +as if they were all on one large private network. + +By using AWS-backed IP pools, egress gateways can be assigned private IPs allowing them to communicate without NAT +within the same VPC, with peered VPCs, and, with peered datacenters. + +### Secondary Elastic Network Interfaces (ENIs) + +Elastic network interfaces are network interfaces that can be added and removed from an instance dynamically. Each +ENI has a primary IP address from the VPC subnet that it belongs to, and it may also have one or more secondary IP +addresses, chosen for the same subnet. While the primary IP address is fixed and cannot be changed, the secondary +IP addresses can be added and removed at runtime. + +To arrange for AWS to route traffic to and from egress gateways, $[prodname] adds _secondary_ Elastic +Network Interfaces (ENIs) to the host. $[prodname] supports two modes for provisioning the +secondary ENIs. The table below describes the trade-offs between **ENI-per-workload** and **Secondary-IP-per-workload** +modes: + +| **ENI-per-workload** (since v3.13) | **Secondary-IP-per-workload** | +| ------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | +| One secondary ENI is attached for each AWS-backed workload. | Secondary ENIs are shared, multiple workloads per ENI. | +| Supports one AWS-backed workload per secondary ENI. | Supports 2-49 AWS-backed workloads per secondary ENI (depending on instance type). | +| ENI Primary IP is set to Workload's IP. | ENI Primary IP chosen from dedicated "host secondary" IP pools. | +| Makes best use of AWS IP space, no need to reserve IPs for hosts. | Requires "host secondary" IPs to be reserved. These cannot be used for workloads. | +| ENI deleted when workload deleted. | ENI retained (ready for next workload to be scheduled). | +| Slower to handle churn/workload mobility. (Creating ENI is slower than assigning IP.) | Faster at handling churn/workload mobility. | + +The number of ENIs that an instance can support and the number of secondary IPs that each ENI can support depends on +the instance type according to [this table](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI). +Note: the table lists the total number of network interfaces and IP addresses but the first interface on the host (the +primary interface) and, in Secondary-IP-per-workload mode, the first IP of each interface (its primary IP) cannot be +used for egress gateways. + +The primary interface cannot be used for egress gateways because it belongs to the VPC subnet that is +in use for Kubernetes hosts; this means that a planned egress gateway IP could get used by AWS as the primary IP of +an instance (for example when scaling up the cluster). + +## Before you begin + +**Required** + +- Calico CNI +- Open port UDP 4790 on the host + +**Not Supported** + +- Amazon VPC CNI + + $[prodname] CNI and IPAM is required. The ability to control the egress gateway’s IP is a feature of $[prodname] CNI and IPAM. AWS VPC CNI does not support that feature, so it is incompatible with egress gateways. + +## How to + +- [Configure IP autodetection](#configure-ip-autodetection) +- [Ensure Kubernetes VPC has free CIDR range](#ensure-kubernetes-vpc-has-free-cidr-range) +- [Create dedicated VPC subnets](#create-dedicated-vpc-subnets) +- [Configure AWS IAM roles](#configure-aws-iam-roles) +- [Configure IP reservations for each VPC subnet](#configure-ip-reservations-for-each-vpc-subnet) +- [Enable egress gateway support](#enable-egress-gateway-support) +- [Enable AWS-backed IP pools](#enable-aws-backed-ip-pools) +- [Configure IP pools backed by VPC subnets](#configure-ip-pools-backed-by-vpc-subnets) +- [Deploy a group of egress gateways](#deploy-a-group-of-egress-gateways) +- [Configure iptables backend for egress gateways](#configure-iptables-backend-for-egress-gateways) +- [Configure namespaces and pods to use egress gateways](#configure-namespaces-and-pods-to-use-egress-gateways) +- [Optionally enable ECMP load balancing](#optionally-enable-ecmp-load-balancing) +- [Verify the feature operation](#verify-the-feature-operation) +- [Control the use of egress gateways](#control-the-use-of-egress-gateways) +- [Policy enforcement for flows via an egress gateway](#policy-enforcement-for-flows-via-an-egress-gateway) +- [Upgrade egress gateways](#upgrade-egress-gateways) + +### Configure IP autodetection + +Since this feature adds additional network interfaces to nodes, it is important to configure $[prodname] to +autodetect the correct primary interface to use for normal pod-to-pod traffic. Otherwise, $[prodname] may +autodetect a newly-added secondary ENI as the main interface, causing an outage. + +For EKS clusters, the default IP autodetection method is `can-reach=8.8.8.8`, which will choose the interface +with a route to `8.8.8.8`; this is typically the interface with a default route, which will be the correct (primary) ENI. +($[prodname] ensures that the secondary ENIs do not have default routes in the main routing table.) + +For other AWS clusters, $[prodname] may default to `firstFound`, which is **not** suitable. + +To examine the autodetection method, check the operator's installation resource: + +```bash +$ kubectl get installations.operator.tigera.io -o yaml default +``` +```yaml noValidation +apiVersion: operator.tigera.io/v1 +kind: Installation +metadata: + ... + name: default + ... +spec: + calicoNetwork: + ... + nodeAddressAutodetectionV4: + firstFound: true +... +``` + +If `nodeAddressAutodetectionV4` is set to `firstFound: true` or is not specified, then you must change it to another method by editing the +resource. The NodeAddressAutodetection options, `canReach` and `cidrs` are suitable. See [Tigera Operator API reference](../../reference/installation/api.mdx). If using the `cidrs` option, set the CIDRs list to include only the +CIDRs from which your primary ENI IPs are chosen (do not include the dedicated VPC subnets chosen below). + +### Ensure Kubernetes VPC has free CIDR range + +For egress gateways to be useful in AWS, we want to assign them IP addresses from a VPC subnet that is in the same AZ +as their host. + +To avoid clashes between AWS IP allocations and $[prodname] IP allocations, it is important that the range of +IP addresses assigned to $[prodname] IP pools is not used by AWS for automatic allocations. In this guide we +assume that you have created a dedicated VPC subnet per Availability Zone (AZ) that is reserved for $[prodname] +and configured not to be used as the default subnet for the AZ. + +If you are creating your cluster and VPC from scratch, plan to subdivide the VPC CIDR into (at least) two VPC subnets +per AZ. One VPC subnet for the Kubernetes (and any other) hosts and one VPC subnet for egress gateways. (The next +section explains the sizing requirements for the egress gateway subnets.) + +If you are adding this feature to an existing cluster, you may find that the existing VPC subnets already cover the +entire VPC CIDR, making it impossible to create a new subnet. If that is the case, you can make more room by +adding a second CIDR to the VPC that is large enough for the new subnets. For information on adding a secondary +CIDR range to a VPC, see [this guide](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Subnets.html#vpc-resize). + +### Create dedicated VPC subnets + +$[prodname] requires a dedicated VPC subnet in each AWS availability zone that you wish to deploy egress +gateways. The subnet must be dedicated to $[prodname] so that AWS will not +use IP addresses from the subnet for other purposes (as this could clash with an egress gateway's IP). When creating the +subnet you should configure it not to be used for instances. + +Some IP addresses from the dedicated subnet are reserved for AWS and $[prodname] internal use: + +- The first four IP addresses in the subnet cannot be used. These are [reserved by AWS for internal use](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Subnets.html#vpc-sizing-ipv4). +- Similarly, the last IP in the subnet (the broadcast address) cannot be used. +- _In **Secondary-IP-per-workload** mode_, $[prodname] requires one IP address from the subnet per secondary ENI + that it provisions (for use as the primary IP address of the ENI). In **ENI-per-workload** mode, this is not required. + + + + +Example for **ENI-per-workload** mode: + +- You anticipate having up to 30 instances running in each availability zone (AZ). +- You intend to use `t3.large` instances, [these are limited to](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI) 3 ENIs per host. +- So, each host can accept 2 secondary ENIs, each of which can handle one egress gateway. +- With 2 ENIs per node and 30 nodes, the part of the cluster in this AZ could handle up to `30 * 2 = 60` egress + gateways. +- AWS reserves 5 IPs from the AWS subnet for internal use, no "host secondary IPs" need to be reserved in this mode. +- Since VPC subnets are allocated by CIDR, a `/25` subnet containing 128 IP addresses would comfortably fit the 5 + reserved IPs as well as the 60 possible gateways (with headroom for more nodes to be added later). + + + + +Example for **Secondary-IP-per-workload** mode: + +- You anticipate having up to 30 instances running in each availability zone (AZ). +- You intend to use `t3.large` instances, [these are limited to](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI) + 3 ENIs per host (one of which is the primary) and each ENI can handle 12 IP addresses, (one of which is the primary). +- So, each host can accept 2 secondary ENIs and each secondary ENI could handle 11 egress gateway pods. +- Each in-use secondary ENI requires one IP from the VPC subnet (up to 60 in this case) and AWS requires 5 IPs to be + reserved so that's up to 65 IPs reserved in total. +- With 2 ENIs and 11 IPs per ENI, the part of the cluster in this AZ could handle up to `30 * 2 * 11 = 660` egress + gateways. +- Since VPC subnets are allocated by CIDR, a `/22` subnet containing 1024 IP addresses would comfortably fit the 65 + reserved IPs as well as the 660 possible gateways. + +$[prodname] allocates ENIs on-demand so each instance will only claim one of those reserved IP addresses when the +first egress gateway is assigned to that node. It will only claim its second IP when that ENI becomes full and then an +extra egress gateway is provisioned. + + + + +### Configure AWS IAM roles + +To provision the required AWS resources, each $[noderunning] pod in your cluster requires the +following IAM permissions to be granted. The permissions can be granted to the node IAM Role itself, or by using +the AWS [IAM roles for service accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) feature to grant the permissions to the +`calico-node` service account. + +- DescribeInstances +- DescribeInstanceTypes +- DescribeNetworkInterfaces +- DescribeSubnets +- DescribeTags +- CreateTags +- AssignPrivateIpAddresses +- UnassignPrivateIpAddresses +- AttachNetworkInterface +- CreateNetworkInterface +- DeleteNetworkInterface +- DetachNetworkInterface +- ModifyNetworkInterfaceAttribute + +The above permissions are similar to those used by the AWS VPC CNI (since both CNIs need to provision the same kinds +of resources). In addition, to support elastic IPs, each $[noderunning] also requires the following permissions: + +- DescribeAddresses +- AssociateAddress +- DisassociateAddress + +### Configure AWS Security Group rules + +To allow egress gateway traffic into the egress gateway pod's host from the client, the ingress rules of the security +group need to be updated. A rule to allow all packets from within the security group must be added to the inbound rules. + +### Configure IP reservations for each VPC subnet + +Since the first four IP addresses and the last IP address in a VPC subnet cannot be used, it is important to +prevent $[prodname] from _trying_ to use them. For each VPC subnet that you plan to use, +ensure that you have an entry in an [IP reservation](../../reference/resources/ipreservation.mdx) for its first +four IP addresses and its final IP address. + +For example, if your chosen VPC subnets are `100.64.0.0/22` and `100.64.4.0/22`, you could create the following +`IPReservation` resource, which covers both VPC subnets (if you're not familiar with CIDR notation, replacing the +`/22` of the original subnet with `/30` is a shorthand for "the first four IP addresses"): + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPReservation +metadata: + name: aws-ip-reservations +spec: + reservedCIDRs: + - 100.64.0.0/30 + - 100.64.3.255 + - 100.64.4.0/30 + - 100.64.7.255 +``` + +### Enable egress gateway support + +In the default **FelixConfiguration**, set the `egressIPSupport` field to `EnabledPerNamespace` or +`EnabledPerNamespaceOrPerPod`, according to the level of support that you need in your cluster. For +support on a per-namespace basis only: + +```bash +kubectl patch felixconfiguration default --type='merge' -p \ + '{"spec":{"egressIPSupport":"EnabledPerNamespace"}}' +``` + +Or for support both per-namespace and per-pod: + +```bash +kubectl patch felixconfiguration default --type='merge' -p \ + '{"spec":{"egressIPSupport":"EnabledPerNamespaceOrPerPod"}}' +``` + +:::note + +- `egressIPSupport` must be the same on all cluster nodes, so you should set them only in the + `default` FelixConfiguration resource. +- The operator automatically enables the required policy sync API in the FelixConfiguration. + +::: + +### Enable AWS-backed IP pools + + + + +To enable **ENI-per-workload** mode, in the default **FelixConfiguration**, set the `awsSecondaryIPSupport` field to +`EnabledENIPerWorkload`: + +```bash +kubectl patch felixconfiguration default --type='merge' -p \ + '{"spec":{"awsSecondaryIPSupport":"EnabledENIPerWorkload"}}' +``` + + + + +To enable **Secondary-IP-per-workload** mode, set the field to `Enabled` (the name `Enabled` predates +the addition of the **ENI-per-workload** mode): + +```bash +kubectl patch felixconfiguration default --type='merge' -p \ + '{"spec":{"awsSecondaryIPSupport":"Enabled"}}' +``` + + + + +You can verify that the setting took effect by examining the Kubernetes Node resources: + +```bash +kubectl describe node +``` + +Should show the new `projectcalico.org/aws-secondary-ipv4` capacity (in the Allocated Resources section). + +#### Changing modes + +You can change between the two modes by: + +- Ensuring that the number of egress gateways on every node is within the limits of the particular mode. i.e. + when switching to **ENI-per-workload** mode, the number of egress gateways must be less than or equal to the number + of secondary ENIs that your instances can handle. +- Editing the setting (using the patch commands above, for example). + +Changing the mode will cause disruption as ENIs must be removed and re-added. + +### Configure IP pools backed by VPC subnets + + + + +In **ENI-per-workload** mode, IP pools are (only) used to subdivide the VPC subnets into small pools used for +particular groups of egress gateways. These IP Pools must have: + +- `awsSubnetID` set to the ID of the relevant VPC subnet. This activates the AWS-backed IP feature for these pools. +- `allowedUse` set to `["Workload"]` to tell $[prodname] IPAM to use those pools for the egress gateway workloads. +- `vxlanMode` and `ipipMode` set to `Never` to disable encapsulation for the egress gateway pods. (`Never` is the default if these fields are not specified.) +- `blockSize` set to 32. This aligns $[prodname] IPAM with the behaviour of the AWS fabric. +- `disableBGPExport` set to `true`. This prevents routing conflicts if your cluster is using IPIP or BGP networking. + +It's also recommended to: + +- Set `nodeSelector` to `"!all()"`. This prevents $[prodname] IPAM from using the pool automatically. It will + only be used for workloads that explicitly name it in the `cni.projectcalico.org/ipv4pools` annotation. + +Continuing the example above, with VPC subnets + +- `100.64.0.0/22` in, say, availability zone west-1 and id `subnet-000000000000000001` +- `100.64.4.0/22` in, say, availability zone west-2 and id `subnet-000000000000000002` + +And, assuming that there are two clusters of egress gateways "red" and "blue" (which in turn serve namespaces "red" +and "blue"), one way to structure the IP pools is to have one IP pool for each group of egress gateways in each +subnet. Then, if a particular egress gateway from the egress gateway cluster is scheduled to one AZ or the other, +it will take an IP from the appropriate pool. + +For the "west-1" availability zone: + +- IP pool "egress-red-west-1", CIDR `100.64.0.4/30` (the first non-reserved /30 CIDR in the VPC subnet). These + addresses will be used for "red" egress gateways in the "west-1" AZ. + +- IP pool "egress-blue-west-1", CIDR `100.64.0.8/30` (the next 4 IPs from the "west-1" subnet). These addresses + will be used for "blue" egress gateways in the "west-1" AZ. + +For the "west-2" availability zone: + +- IP pool "egress-red-west-2", CIDR `100.64.4.4/30` (the first non-reserved /30 CIDR in the VPC subnet). These + addresses will be used for "red" egress gateways in the "west-2" AZ. + +- IP pool "egress-blue-west-2", CIDR `100.64.4.8/30` (the next 4 IPs from the "west-2" subnet). These addresses + will be used for "blue" egress gateways in the "west-2" AZ. + +Converting this to `IPPool` resources: + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: egress-red-west-1 +spec: + cidr: 100.64.0.4/30 + allowedUses: ['Workload'] + awsSubnetID: subnet-000000000000000001 + blockSize: 32 + nodeSelector: '!all()' + disableBGPExport: true +--- +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: egress-blue-west-1 +spec: + cidr: 100.64.0.8/30 + allowedUses: ['Workload'] + awsSubnetID: subnet-000000000000000001 + blockSize: 32 + nodeSelector: '!all()' + disableBGPExport: true +--- +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: egress-red-west-2 +spec: + cidr: 100.64.4.4/30 + allowedUses: ['Workload'] + awsSubnetID: subnet-000000000000000002 + blockSize: 32 + nodeSelector: '!all()' + disableBGPExport: true +--- +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: egress-blue-west-2 +spec: + cidr: 100.64.4.8/30 + allowedUses: ['Workload'] + awsSubnetID: subnet-000000000000000002 + blockSize: 32 + nodeSelector: '!all()' + disableBGPExport: true +``` + + + + +In **Secondary-IP-per-workload** mode, IP pools are used to subdivide the VPC subnets as follows: + +- One medium-sized IP pool per-Subnet reserved for $[prodname] to use for the _primary_ IP addresses of its _secondary_ ENIs. + These pools must have: + + - `awsSubnetID` set to the ID of the relevant VPC subnet. This activates the AWS-backed IP feature for these pools. + - `allowedUse` set to `["HostSecondaryInterface"]` to reserve them for this purpose. + - `blockSize` set to 32. This aligns $[prodname] IPAM with the behaviour of the AWS fabric. + - `vxlanMode` and `ipipMode` set to `Never`. (`Never` is the default if these fields are not specified.) + - `disableBGPExport` set to `true`. This prevents routing conflicts if your cluster is using IPIP or BGP networking. + +- Small pools used for particular groups of egress gateways. These must have: + + - `awsSubnetID` set to the ID of the relevant VPC subnet. This activates the AWS-backed IP feature for these pools. + - `allowedUse` set to `["Workload"]` to tell $[prodname] IPAM to use those pools for the egress gateway workloads. + - `vxlanMode` and `ipipMode` set to `Never` to disable encapsulation for the egress gateway pods. (`Never` is the default if these fields are not specified.) + - `blockSize` set to 32. This aligns $[prodname] IPAM with the behaviour of the AWS fabric. + - `disableBGPExport` set to `true`. This prevents routing conflicts if your cluster is using IPIP or BGP networking. + + It's also recommended to: + + - Set `nodeSelector` to `"!all()"`. This prevents $[prodname] IPAM from using the pool automatically. It will + only be used for workloads that explicitly name it in the `cni.projectcalico.org/ipv4pools` annotation. + +Continuing the example above, with VPC subnets + +- `100.64.0.0/22` in, say, availability zone west-1 and id `subnet-000000000000000001` +- `100.64.4.0/22` in, say, availability zone west-2 and id `subnet-000000000000000002` + +And, assuming that there are two clusters of egress gateways "red" and "blue" (which in turn serve namespaces "red" +and "blue"), one way to structure the IP pools is to have a "hosts" IP pool in each VPC subnet and one IP pool for each +group of egress gateways in each subnet. Then, if a particular egress gateway from the egress gateway cluster is +scheduled to one AZ or the other, it will take an IP from the appropriate pool. + +For the "west-1" availability zone: + +- IP pool "hosts-west-1", CIDR `100.64.0.0/25` (the first 128 addresses in the "west-1" VPC subnet). + + - We'll reserve these addresses for hosts to use. + - `100.64.0.0/25` covers the addresses from `100.64.0.0` to `100.64.0.127` (but addresses `100.64.0.0` to `100.64.0.3` + were reserved above). + +- IP pool "egress-red-west-1", CIDR `100.64.0.128/30` (the next 4 IPs from the "west-1" subnet). + + - These addresses will be used for "red" egress gateways in the "west-1" AZ. + +- IP pool "egress-blue-west-1", CIDR `100.64.0.132/30` (the next 4 IPs from the "west-1" subnet). + + - These addresses will be used for "blue" egress gateways in the "west-1" AZ. + +For the "west-2" availability zone: + +- IP pool "hosts-west-2", CIDR `100.64.4.0/25` (the first 128 addresses in the "west-2" VPC subnet). + + - `100.64.4.0/25` covers the addresses from `100.64.4.0` to `100.64.4.127` (but addresses `100.64.4.0` to `100.64.4.3` + were reserved above). + +- IP pool "egress-red-west-2", CIDR `100.64.4.128/30` (the next 4 IPs from the "west-2" subnet). + + - These addresses will be used for "red" egress gateways in the "west-2" AZ. + +- IP pool "egress-blue-west-2", CIDR `100.64.4.132/30` (the next 4 IPs from the "west-2" subnet). + + - These addresses will be used for "blue" egress gateways in the "west-2" AZ. + +Converting this to `IPPool` resources: + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: hosts-west-1 +spec: + cidr: 100.64.0.0/25 + allowedUses: ['HostSecondaryInterface'] + awsSubnetID: subnet-000000000000000001 + blockSize: 32 + disableBGPExport: true +--- +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: egress-red-west-1 +spec: + cidr: 100.64.0.128/30 + allowedUses: ['Workload'] + awsSubnetID: subnet-000000000000000001 + blockSize: 32 + nodeSelector: '!all()' + disableBGPExport: true +--- +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: egress-blue-west-1 +spec: + cidr: 100.64.0.132/30 + allowedUses: ['Workload'] + awsSubnetID: subnet-000000000000000001 + blockSize: 32 + nodeSelector: '!all()' + disableBGPExport: true +--- +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: hosts-west-2 +spec: + cidr: 100.64.4.0/25 + allowedUses: ['HostSecondaryInterface'] + awsSubnetID: subnet-000000000000000002 + blockSize: 32 + disableBGPExport: true +--- +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: egress-red-west-2 +spec: + cidr: 100.64.4.128/30 + allowedUses: ['Workload'] + awsSubnetID: subnet-000000000000000002 + blockSize: 32 + nodeSelector: '!all()' + disableBGPExport: true +--- +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: egress-blue-west-2 +spec: + cidr: 100.64.4.132/30 + allowedUses: ['Workload'] + awsSubnetID: subnet-000000000000000002 + blockSize: 32 + nodeSelector: '!all()' + disableBGPExport: true +``` + + + + +### Deploy a group of egress gateways + +Use an egress gateway custom resource to deploy a group of egress gateways. + +Using the example of the "red" egress gateway cluster, we use several features of Kubernetes and $[prodname] +in tandem to get a cluster of egress gateways that spans both availability zones and uses AWS-backed IP addresses: + +```bash +kubectl apply -f - < +# - +# timeoutSeconds: 15 +# intervalSeconds: 5 +# httpProbe: +# urls: +# - +# - +# timeoutSeconds: 30 +# intervalSeconds: 10 + aws: + nativeIP: Enabled + template: + metadata: + labels: + egress-code: red + spec: + nodeSelector: + kubernetes.io/os: linux + terminationGracePeriodSeconds: 0 + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: "topology.kubernetes.io/zone" + whenUnsatisfiable: "DoNotSchedule" + labelSelector: + matchLabels: + egress-code: red +EOF +``` + +- `replicas: 2` tells Kubernetes to schedule two egress gateways in the "red" cluster. +- ipPools tells $[prodname] IPAM to use one of the "red" IP pools: + + ```yaml + ipPools: + - name: "egress-red-west-1" + - name: "egress-red-west-2" + ``` + Depending on which AZ the pod is scheduled in, $[prodname] IPAM will automatically ignore IP pools that + are backed by AWS subnets that are not in the local AZ. + + External services and appliances can recognise "red" traffic because it will all come from the CIDRs of the "red" + IP pools. + +- When nativeIP is enabled, IPPools must be AWS-backed. It also tells Kubernetes to only schedule the gateway to a node + with available AWS IP capacity: + + ```yaml + aws: + nativeIP: Enabled + ``` + +- The following [topology spread constraint](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) + ensures that Kubernetes spreads the Egress gateways evenly between AZs (assuming that your nodes are labeled with + the expected [well-known label](https://kubernetes.io/docs/reference/labels-annotations-taints/#topologykubernetesiozone) + `topology.kubernetes.io/zone`): + + ```yaml + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + egress-code: red + ``` + +- The labels are arbitrary. You can choose whatever names and values are convenient for your cluster's Namespaces and Pods to refer to in their egress selectors. + If labels are not specified, a default label `projectcalico.org/egw`:`name` will be added by the Tigera Operator. + +- icmpProbe may be used to specify the Probe IPs, ICMP interval and timeout in seconds. `ips` if set, the + egress gateway pod will probe each IP periodically using an ICMP ping. If all pings fail then the egress + gateway will report non-ready via its health port. `intervalSeconds` controls the interval between probes. + `timeoutSeconds` controls the timeout before reporting non-ready if no probes succeed. + + ```yaml + icmpProbe: + ips: + - + - + timeoutSeconds: 20 + intervalSeconds: 10 + ``` + +- httpProbe may be used to specify the Probe URLs, HTTP interval and timeout in seconds. `urls` if set, the + egress gateway pod will probe each external service periodically. If all probes fail then the egress + gateway will report non-ready via its health port. `intervalSeconds` controls the interval between probes. + `timeoutSeconds` controls the timeout before reporting non-ready if all probes are failing. + + ```yaml + httpProbe: + urls: + - + - + timeoutSeconds: 30 + intervalSeconds: 10 + ``` +- Please refer to the [operator reference docs](../../reference/installation/api.mdx) for details about the egress gateway resource type. + +:::note + +- It is advisable to have more than one egress gateway per group, so that the egress IP function + continues if one of the gateways crashes or needs to be restarted. When there are multiple + gateways in a group, outbound traffic from the applications using that group is load-balanced + across the available gateways. The number of `replicas` specified must be less than or equal + to the number of free IP addresses in the IP Pool. +- IPPool can be specified either by its name (e.g. `-name: egress-ippool-1`) or by its CIDR (e.g. `-cidr: 10.10.10.0/31`). +- The labels are arbitrary. You can choose whatever names and values are convenient for + your cluster's Namespaces and Pods to refer to in their egress selectors. + The health port `8080` is used by: +- The Kubernetes `readinessProbe` to expose the status of the egress gateway pod (and any ICMP/HTTP + probes). +- Remote pods to check if the egress gateway is "ready". Only "ready" egress + gateways will be used for remote client traffic. This traffic is automatically allowed by $[prodname] and + no policy is required to allow it. $[prodname] only sends probes to egress gateway pods that have a named + "health" port. This ensures that during an upgrade, health probes are only sent to upgraded egress gateways. + +::: + +### Configure iptables backend for egress gateways + +The Tigera Operator configures egress gateways to use the same iptables backend as `calico-node`. +To modify the iptables backend for egress gateways, you must change the `iptablesBackend` field in the [Felix configuration](../../reference/resources/felixconfig.mdx). + +### Configure namespaces and pods to use egress gateways + +You can configure namespaces and pods to use an egress gateway by: +* annotating the namespace or pod +* applying an egress gateway policy to the namespace or pod. + +Using an egress gateway policy is more complicated, but it allows advanced use cases. + +#### Configure a namespace or pod to use an egress gateway (annotation method) + +In a $[prodname] deployment, the Kubernetes namespace and pod resources honor annotations that +tell that namespace or pod to use particular egress gateways. These annotations are selectors, and +their meaning is "the set of pods, anywhere in the cluster, that match those selectors". + +So, to configure all the pods in a namespace to use the egress gateways that are +labelled with `egress-code: red`, you would annotate that namespace like this: + +```bash +kubectl annotate ns egress.projectcalico.org/selector="egress-code == 'red'" +``` + +By default, that selector can only match egress gateways in the same namespace. To select gateways +in a different namespace, specify a `namespaceSelector` annotation as well, like this: + +```bash +kubectl annotate ns egress.projectcalico.org/namespaceSelector="kubernetes.io/metadata.name == 'default'" +``` + +Egress gateway annotations have the same [syntax and range of expressions](../../reference/resources/networkpolicy.mdx#selector) as the selector fields in +$[prodname] [network policy](../../reference/resources/networkpolicy.mdx#entityrule). + +To configure a specific Kubernetes Pod to use egress gateways, specify the same annotations when +creating the pod. For example: + +```bash +kubectl apply -f - < egress.projectcalico.org/egressGatewayPolicy="egw-policy1" +``` + +To configure a specific Kubernetes pod to use the same policy, specify the same annotations when +creating the pod. +For example: + +```bash +kubectl apply -f - < -n -- nc 8089 ` should be the IP address of the netcat server. + +Then, if you check the logs or output of the netcat server, you should see: + +``` +Connection from received +``` + +with `` being one of the IPs of the egress IP pool that you provisioned. + +### Control the use of egress gateways + +If a cluster ascribes special meaning to traffic flowing through egress gateways, it will be +important to control when cluster users can configure their pods and namespaces to use them, so that +non-special pods cannot impersonate the special meaning. + +If namespaces in a cluster can only be provisioned by cluster admins, one option is to enable egress +gateway function only on a per-namespace basis. Then only cluster admins will be able to configure +any egress gateway usage. + +Otherwise -- if namespace provisioning is open to users in general, or if it's desirable for egress +gateway function to be enabled both per-namespace and per-pod -- a [Kubernetes admission controller](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/) + will be +needed. This is a task for each deployment to implement for itself, but possible approaches include +the following. + +1. Decide whether a given Namespace or Pod is permitted to use egress annotations at all, based on + other details of the Namespace or Pod definition. + +1. Evaluate egress annotation selectors to determine the egress gateways that they map to, and + decide whether that usage is acceptable. + +1. Impose the cluster's own bespoke scheme for a Namespace or Pod to identify the egress gateways + that it wants to use, less general than $[prodname]'s egress annotations. Then the + admission controller would police those bespoke annotations (that that cluster's users could + place on Namespace or Pod resources) and either reject the operation in hand, or allow it + through after adding the corresponding $[prodname] egress annotations. + +### Policy enforcement for flows via an egress gateway + +For an outbound connection from a client pod, via an egress gateway, to a destination outside the +cluster, there is more than one possible enforcement point for policy: + +The path of the traffic through policy is as follows: + +1. Packet leaves the client pod and passes through its egress policy. +2. The packet is encapsulated by the client pod's host and sent to the egress gateway +3. The encapsulated packet is sent from the host to the egress gateway pod. +4. The egress gateway pod de-encapsulates the packet and sends the packet out again with its own address. +5. The packet leaves the egress gateway pod through its egress policy. + +To ensure correct operation, (as of v3.15) the encapsulated traffic between host and egress gateway is auto-allowed by +$[prodname] and other ingress traffic is blocked. That means that there are effectively two places where +policy can be applied: + +1. on egress from the client pod +2. on egress from the egress gateway pod (see limitations below). + +The policy applied at (1) is the most powerful since it implicitly sees the original source of the traffic (by +virtue of being attached to that original source). It also sees the external destination of the traffic. + +Since an egress gateway will never originate its own traffic, one option is to rely on policy applied at (1) and +to allow all traffic to at (2) (either by applying no policy or by applying an "allow all"). + +Alternatively, for maximum "defense in depth" applying policy at both (1) and (2) provides extra protection should +the policy at (1) be disabled or bypassed by an attacker. Policy at (2) has the following limitations: + +- [Domain-based policy](../../network-policy/domain-based-policy.mdx) is not supported at egress from egress + gateways. It will either fail to match the expected traffic, or it will work intermittently if the egress gateway + happens to be scheduled to the same node as its clients. This is because any DNS lookup happens at the client pod. + By the time the policy reaches (2) the DNS information is lost and only the IP addresses of the traffic are available. + +- The traffic source will appear to be the egress gateway pod, the source information is lost in the address + translation that occurs inside the egress gateway pod. + +That means that policies at (2) will usually take the form of rules that match only on destination port and IP address, +either directly in the rule (via a CIDR match) or via a (non-domain based) NetworkSet. Matching on source has little +utility since the IP will always be the egress gateway and the port of translated traffic is not always preserved. + +:::note + +Since v3.15.0, $[prodname] also sends health probes to the egress gateway pods from the nodes where +their clients are located. In iptables mode, this traffic is auto-allowed at egress from the host and ingress +to the egress gateway. In eBPF mode, the probe traffic can be blocked by policy, so you must ensure that this traffic is allowed; this should be fixed in an upcoming +patch release. + +::: + +## Upgrade egress gateways + +From v3.16, egress gateway deployments are managed by the Tigera Operator. + +- When upgrading from a pre-v3.16 release, no automatic upgrade will occur. To upgrade a pre-v3.16 egress gateway deployment, + create an equivalent EgressGateway resource with the same namespace and the same name as mentioned [above](#deploy-a-group-of-egress-gateways); + the operator will then take over management of the old Deployment resource, replacing it with the upgraded version. + +- Use `kubectl apply` to create the egress gateway resource. Tigera Operator will read the newly created resource and wait + for the other $[prodname] components to be upgraded. Once the other $[prodname] components are upgraded, Tigera Operator + will upgrade the existing egress gateway deployment with the new image. + +By default, upgrading egress gateways will sever any connections that are flowing through them. To minimise impact, +the egress gateway feature supports some advanced options that give feedback to affected pods. For more details see +the [egress gateway maintenance guide](egress-gateway-maintenance.mdx). + +## Additional resources + +Please see also: + +- The `egressIP...` and `aws...` fields of the [FelixConfiguration resource](../../reference/resources/felixconfig.mdx#spec). +- [Troubleshooting egress gateways](troubleshoot.mdx). +- [Additional configuration for egress gateway maintenance](egress-gateway-maintenance.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/egress/egress-gateway-azure.mdx b/calico-cloud_versioned_docs/version-24-1/networking/egress/egress-gateway-azure.mdx new file mode 100644 index 0000000000..166e385eb7 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/egress/egress-gateway-azure.mdx @@ -0,0 +1,775 @@ +--- +description: Route selected application traffic out of a Calico Cloud connected cluster through egress gateways with native Azure VNet IPs. +--- + +# Configure egress gateways, Azure + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## Big picture + +Control the source IP address seen by external services/appliances by routing the traffic from certain pods +through egress gateways. Use native VNet subnet IP addresses for the egress gateways so that the IPs are valid in the Azure fabric. + +## Value + +Controlling the source IP seen when traffic leaves the cluster allows groups of pods to be identified +by external firewalls, appliances and services (even as the groups are scaled up/down or pods restarted). +$[prodname] controls the source IP by directing traffic through one or more "egress gateway" pods, which +change the source IP of the traffic to their own IP. The egress gateways used can be chosen at the pod or namespace +scope allowing for flexibility in how the cluster is seen from outside. + +In Azure, egress gateway source IP addresses are chosen from an arbitrary user-defined IP pool +using $[prodname] IPAM. egress gateway pods use dedicated IPPools to use as source IP addresses, which enables static configuration +of external appliances. + +## Concepts + +### Egress gateway + +An egress gateway acts as a transit pod for the outbound application traffic that is configured to +use it. As traffic leaving the cluster passes through the egress gateway, its source IP is changed +to that of the egress gateway pod, and the traffic is then forwarded on. + +### Source IP + +When an outbound application flow leaves the cluster, its IP packets will have a source IP. +This begins as the pod IP of the pod that originated the flow, then: + +- _If no egress gateway is configured_ and the pod IP came from an [IP pool](../../reference/resources/ippool.mdx) +with `natOutgoing: true`, the node hosting the pod will change the source IP to its own as the +traffic leaves the host. This allows the pod to communicate with external service even though the +external network is unaware of the pod's IP. + +- _If the pod is configured with an egress gateway_, the traffic is first forwarded to the egress gateway, which +changes the source IP to its own and then sends the traffic on. To function correctly, egress gateways +should have IPs from an IP pool with `natOutgoing: false`, meaning their host forwards the packet onto +the network without changing the source IP again. Since the egress gateway's IP is visible to +the underlying network fabric, the fabric must be configured to know about the egress gateway's +IP and to send response traffic back to the same host. + +### Azure VNets and subnets + +An Azure VNet is a virtual network that is, by default, logically isolated from other VNets. Each VNet has one or more +(often large) CIDR blocks associated with it (for example `10.0.0.0/16`). In general, VNet CIDRs may overlap, but only +if the VNets remain isolated. Azure allows VNets to be peered with each other through VNet Peerings. VNets can only be +peered if *none* of their associated CIDRs overlap. + +Each VNet has one or more VNet subnets associated with it, each subnet owns a non-overlapping part of one of the +VNet's CIDR blocks. VNet subnets span all availability zones in a region, so it is possible to distribute egress gateway +resources across availability zones in a region without the need to repeat the configuration per-AZ. + +### Azure VNet and ExpressRoute peerings + +Azure [VNet Peerings](https://learn.microsoft.com/en-us/azure/virtual-network/virtual-network-peering-overview) allow multiple VNets to be +connected together. Similarly, [ExpressRoute](https://learn.microsoft.com/en-us/azure/expressroute/expressroute-introduction) +allows external datacenters to be connected to an Azure VNet. Peered VPCs and datacenters communicate using private IPs +as if they were all on one large private network. + +By advertising routes to Azure fabric using Azure Route Servers, egress gateways can be assigned private IPs allowing them to +communicate without NAT within the same VPC, with peered VPCs, and, with peered datacenters. + +### Azure Route Server + +Azure [Route Server](https://learn.microsoft.com/en-us/azure/route-server/) is a managed networking service that allows +a network virtual appliance, like $[prodname], to dynamically configure Azure fabric by exchanging routing information +using Border Gateway Protocol (BGP). $[prodname] can establish BGP sessions with an Azure Route Server in a VNet +to advertise the IPs of egress gateways to that VNet. The learned routes are then propagated to the rest of VNets through +VNet peering, or to external datacenters through ExpressRoute, allowing communication with egress gateway. + +## Before you begin + +**Required** + +- Calico CNI +- Open port UDP 4790 on the host + +**Not Supported** + +- Azure VNet CNI + +$[prodname] CNI and IPAM are required. The ability to control the egress gateway’s IP is a feature of $[prodname] CNI and IPAM. Azure VNet CNI does not support that feature, so it is incompatible with egress gateways. + +## How to + +- [Choose route reflectors](#choose-route-reflectors) +- [Create Azure Route Server](#create-azure-route-server) +- [Disable the default BGP node-to-node mesh](#disable-the-default-bgp-node-to-node-mesh) +- [Enable BGP](#enable-bgp) +- [Provision an egress IP pool](#provision-an-egress-ip-pool) +- [(Optional) Limit number of route advertisement](#optional-limit-number-of-route-advertisements) +- [Configure route reflector](#configure-route-reflectors) +- [Enable egress gateway support](#enable-egress-gateway-support) +- [Deploy a group of egress gateways](#deploy-a-group-of-egress-gateways) +- [Configure iptables backend for egress gateways](#configure-iptables-backend-for-egress-gateways) +- [Configure namespaces and pods to use egress gateways](#configure-namespaces-and-pods-to-use-egress-gateways) +- [(Optional) Enable ECMP load balancing](#optional-enable-ecmp-load-balancing) +- [Verify the feature operation](#verify-the-feature-operation) +- [Control the use of egress gateways](#control-the-use-of-egress-gateways) +- [Policy enforcement for flows via an egress gateway](#policy-enforcement-for-flows-via-an-egress-gateway) + +### Choose route reflectors + +It is possible to establish BGP connections between all $[prodname] nodes and Azure Route Servers, but +to avoid hitting Azure Route Server peers [limit](https://learn.microsoft.com/en-us/azure/route-server/overview#route-server-limits), +it is better to select some nodes as route reflectors and set up BGP connections between those and Azure Route Server. +The number of route reflectors depends on the cluster size, but it is recommended to have at least 3 at all times, +and Azure Route Servers supports up to 8 peers in a VNet. + +### Create Azure Route Server + +Deploy Azure [Route Server](https://learn.microsoft.com/en-us/azure/route-server/) in the VNet (hub or spoke VNet) that routes +egress addresses. Then, add the selected route reflectors as peers to the Azure Route Server. + +:::note + +- The BGP connections between Calico route reflectors and Azure Route Servers are critical for the functionality +of egress gateways. It is important to maintain route reflectors with care, and to make sure there are always +enough healthy route reflectors. +- If possible, assign a static address to the route reflectors so after reboots the same address is kept. +- In AKS, it is [recommended](https://learn.microsoft.com/en-us/azure/aks/use-system-pools?tabs=azure-cli#system-and-user-node-pools) +to run applications in user node pools and leave system node pools for running critical system pods. +The nodes in system node pools are perfect Route Reflector candidates. + +::: + +### Disable the default BGP node-to-node mesh + +The default $[prodname] **node-to-node BGP mesh** may be turned off to enable other BGP topologies. +To do this, modify the default **BGP configuration** resource. + +Run the following command to disable the BGP full-mesh: + +```bash +kubectl apply -f - < projectcalico.org/RouteReflectorClusterID=244.0.0.1 +``` + +Typically, you will want to label this node to indicate that it is a route reflector, allowing it to be easily selected by a BGPPeer resource. You can do this with kubectl. For example: + +```bash +kubectl label node route-reflector=true +``` + +Now it is easy to configure route reflector nodes to peer with each other and other non-route-reflector nodes using label selectors. For example: + +```yaml +kind: BGPPeer +apiVersion: projectcalico.org/v3 +metadata: + name: peer-with-route-reflectors +spec: + nodeSelector: all() + peerSelector: route-reflector == 'true' +``` + +Finally, Add the IP addresses of Azure Route Servers, usually there are two, as peers to route reflectors. For example, if the $[prodname] +cluster is in a subnet with 10.224.0.0 as network address, the 10.224.0.1 is, [by definition](https://learn.microsoft.com/en-us/azure/virtual-network/virtual-networks-faq#are-there-any-restrictions-on-using-ip-addresses-within-these-subnets), the gateway and Azure Route Servers are deployed with +ASN 65515 and IP addresses of 10.225.0.4 and 10.225.0.5, create two **BGPPeer** resources: + +```yaml +apiVersion: projectcalico.org/v3 +kind: BGPPeer +metadata: + name: azure-route-server-a +spec: + peerIP: 10.225.0.4 + reachableBy: 10.224.0.1 + asNumber: 65515 + keepOriginalNextHop: true + nodeSelector: route-reflector == 'true' + filters: + - export-egress-ips +--- +apiVersion: projectcalico.org/v3 +kind: BGPPeer +metadata: + name: azure-route-server-b +spec: + peerIP: 10.225.0.5 + reachableBy: 10.224.0.1 + asNumber: 65515 + keepOriginalNextHop: true + nodeSelector: route-reflector == 'true' + filters: + - export-egress-ips +``` + +:::note + +- Adding `routeReflectorClusterID` to a node spec will remove it from the node-to-node mesh immediately, tearing down the +existing BGP sessions. Adding the BGP peering will bring up new BGP sessions. This will cause a short (about 2 seconds) +disruption to data plane traffic of workloads running in the nodes where this happens. To avoid this, make sure no +workloads are running on the nodes, by provisioning new nodes or by running `kubectl drain` on the node (which may +itself cause a disruption as workloads are drained). +- It is important to set `keepOriginalNextHop: true` since route reflectors advertise routes on behalf of other nodes. +Advertised routes to Azure Route Servers should have the original next hop otherwise the return packets will be sent to +route reflectors, and get dropped. +- It is mandatory to set `reachableBy` field set to the gateway of the subnet $[prodname] cluster is running in for +peering with Azure Route Servers to prevent BGP connection flapping. See [BGP peer](../../reference/resources/bgppeer.mdx) for more information. +- Including `filters` to apply the BGP filter configured in the previous section, is optional. +::: + +### Enable egress gateway support + +In the default **FelixConfiguration**, set the `egressIPSupport` field to `EnabledPerNamespace` or +`EnabledPerNamespaceOrPerPod`, according to the level of support that you need in your cluster. For +support on a per-namespace basis only: + +```bash +kubectl patch felixconfiguration default --type='merge' -p \ + '{"spec":{"egressIPSupport":"EnabledPerNamespace"}}' +``` + +Or for support both per-namespace and per-pod: + +```bash +kubectl patch felixconfiguration default --type='merge' -p \ + '{"spec":{"egressIPSupport":"EnabledPerNamespaceOrPerPod"}}' +``` + +:::note + +- `egressIPSupport` must be the same on all cluster nodes, so you should set them only in the `default` FelixConfiguration resource. +- The operator automatically enables the required policy sync API in the FelixConfiguration. + +::: + +### Deploy a group of egress gateways + +Use a Kubernetes Deployment to deploy a group of egress gateways. + +Using the example of the "red" egress gateway cluster, we use several features of Kubernetes and $[prodname] +in tandem to get a cluster of egress gateways that uses the user defined IP addresses: + +```bash +kubectl apply -f - < +# - +# timeoutSeconds: 15 +# intervalSeconds: 5 +# httpProbe: +# urls: +# - +# - +# timeoutSeconds: 30 +# intervalSeconds: 10 + template: + metadata: + labels: + egress-code: red + spec: + terminationGracePeriodSeconds: 0 + nodeSelector: + kubernetes.io/os: linux + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + egress-code: red +EOF +``` + +* `replicas: 2` tells Kubernetes to schedule two egress gateways in the "red" cluster. + +* This annotation tells $[prodname] IPAM to use one of the "egress-ip-red-pool" IP pool. + +External services and appliances can recognise "red" traffic because it will all come from the CIDRs of the "red" +IP pool. + +* The following [topology spread constraint](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) +ensures that Kubernetes spreads the Egress gateways evenly between AZs (assuming that your nodes are labeled with +the expected [well-known label](https://kubernetes.io/docs/reference/labels-annotations-taints/#topologykubernetesiozone) +`topology.kubernetes.io/zone`): + +```yaml +topologySpreadConstraints: +- maxSkew: 1 + topologyKey: topology.kubernetes.io/zone + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + egress-code: red + ``` + +* The labels are arbitrary. You can choose whatever names and values are convenient for your cluster's Namespaces and Pods to refer to in their egress selectors. +If labels are not specified, a default label `projectcalico.org/egw`:`name` will be added by the Tigera Operator. + +* icmpProbe may be used to specify the Probe IPs, ICMP interval and timeout in seconds. `ips` if set, the +egress gateway pod will probe each IP periodically using an ICMP ping. If all pings fail then the egress +gateway will report non-ready via its health port. `intervalSeconds` controls the interval between probes. +`timeoutSeconds` controls the timeout before reporting non-ready if no probes succeed. + +```yaml +icmpProbe: + ips: + - + - + timeoutSeconds: 20 + intervalSeconds: 10 + ``` + +* httpProbe may be used to specify the Probe URLs, HTTP interval and timeout in seconds. `urls` if set, the +egress gateway pod will probe each external service periodically. If all probes fail then the egress +gateway will report non-ready via its health port. `intervalSeconds` controls the interval between probes. +`timeoutSeconds` controls the timeout before reporting non-ready if all probes are failing. + +```yaml +httpProbe: + urls: + - + - + timeoutSeconds: 30 + intervalSeconds: 10 + ``` +- Please refer to the [operator reference docs](../../reference/installation/api.mdx) for details about the egress gateway resource type. + +:::note + +* It is advisable to have more than one egress gateway per group, so that the egress IP function +continues if one of the gateways crashes or needs to be restarted. When there are multiple +gateways in a group, outbound traffic from the applications using that group is load-balanced +across the available gateways. The number of `replicas` specified must be less than or equal +to the number of free IP addresses in the IP Pool. + +* IPPool can be specified either by its name (e.g. `-name: egress-ip-red-pool`) or by its CIDR (e.g. `-cidr: 10.10.10.0/30`). + +* The labels are arbitrary. You can choose whatever names and values are convenient for +your cluster's Namespaces and Pods to refer to in their egress selectors. + +The health port `8080` is used by: + +* The Kubernetes `readinessProbe` to expose the status of the egress gateway pod (and any ICMP/HTTP probes). + +* Remote pods to check if the egress gateway is "ready". Only "ready" egress +gateways will be used for remote client traffic. This traffic is automatically allowed by $[prodname] and +no policy is required to allow it. $[prodname] only sends probes to egress gateway pods that have a named +"health" port. This ensures that during an upgrade, health probes are only sent to upgraded egress gateways. +::: + +### Configure iptables backend for egress gateways + +The Tigera Operator configures egress gateways to use the same iptables backend as `calico-node`. +To modify the iptables backend for egress gateways, you must change the `iptablesBackend` field in the [Felix configuration](../../reference/resources/felixconfig.mdx). + +### Configure namespaces and pods to use egress gateways + +You can configure namespaces and pods to use an egress gateway by: +* annotating the namespace or pod +* applying an egress gateway policy to the namespace or pod. + +Using an egress gateway policy is more complicated, but it allows advanced use cases. + +#### Configure a namespace or pod to use an egress gateway (annotation method) + +In a $[prodname] deployment, the Kubernetes namespace and pod resources honor annotations that +tell that namespace or pod to use particular egress gateways. These annotations are selectors, and +their meaning is "the set of pods, anywhere in the cluster, that match those selectors". + +So, to configure all the pods in a namespace to use the egress gateways that are +labelled with `egress-code: red`, you would annotate that namespace like this: + +```bash +kubectl annotate ns egress.projectcalico.org/selector="egress-code == 'red'" +``` + +By default, that selector can only match egress gateways in the same namespace. To select gateways +in a different namespace, specify a `namespaceSelector` annotation as well, like this: + +```bash +kubectl annotate ns egress.projectcalico.org/namespaceSelector="kubernetes.io/metadata.name == 'default'" +``` + +Egress gateway annotations have the same [syntax and range of expressions](../../reference/resources/networkpolicy.mdx#selector) as the selector fields in +$[prodname] [network policy](../../reference/resources/networkpolicy.mdx#entityrule). + +To configure a specific Kubernetes Pod to use egress gateways, specify the same annotations when +creating the pod. For example: + +```bash +kubectl apply -f - < egress.projectcalico.org/egressGatewayPolicy="egw-policy1" +``` + +To configure a specific Kubernetes pod to use the same policy, specify the same annotations when +creating the pod. +For example: + +```bash +kubectl apply -f - < +spec: + containers: + - name: alpine + image: alpine + command: ["/bin/sleep"] + args: ["infinity"] +EOF +``` + +Now you can use `kubectl exec` to initiate an outbound connection from that pod: + +```bash +kubectl exec -n -- nc 8089 ` should be the IP address of the netcat server. + +Then, if you check the logs or output of the netcat server, you should see: + +``` +Connection from received +``` + +with `` being one of the IPs of the egress IP pool that you provisioned. + +### Control the use of egress gateways + +If a cluster ascribes special meaning to traffic flowing through egress gateways, it will be +important to control when cluster users can configure their pods and namespaces to use them, so that +non-special pods cannot impersonate the special meaning. + +If namespaces in a cluster can only be provisioned by cluster admins, one option is to enable egress +gateway function only on a per-namespace basis. Then only cluster admins will be able to configure +any egress gateway usage. + +Otherwise -- if namespace provisioning is open to users in general, or if it's desirable for egress +gateway function to be enabled both per-namespace and per-pod -- a [Kubernetes admission controller](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/) + will be +needed. This is a task for each deployment to implement for itself, but possible approaches include +the following. + +1. Decide whether a given Namespace or Pod is permitted to use egress annotations at all, based on +other details of the Namespace or Pod definition. + +1. Evaluate egress annotation selectors to determine the egress gateways that they map to, and +decide whether that usage is acceptable. + +1. Impose the cluster's own bespoke scheme for a Namespace or Pod to identify the egress gateways +that it wants to use, less general than $[prodname]'s egress annotations. Then the +admission controller would police those bespoke annotations (that that cluster's users could +place on Namespace or Pod resources) and either reject the operation in hand, or allow it +through after adding the corresponding $[prodname] egress annotations. + +#### Policy enforcement for flows via an egress gateway + +For an outbound connection from a client pod, via an egress gateway, to a destination outside the +cluster, there is more than one possible enforcement point for policy: + +The path of the traffic through policy is as follows: + +1. Packet leaves the client pod and passes through its egress policy. +2. The packet is encapsulated by the client pod's host and sent to the egress gateway +3. The encapsulated packet is sent from the host to the egress gateway pod. +4. The egress gateway pod de-encapsulates the packet and sends the packet out again with its own address. +5. The packet leaves the egress gateway pod through its egress policy. + +To ensure correct operation, (as of v3.15) the encapsulated traffic between host and egress gateway is auto-allowed by +$[prodname] and other ingress traffic is blocked. That means that there are effectively two places where +policy can be applied: + +1. on egress from the client pod +2. on egress from the egress gateway pod (see limitations below). + +The policy applied at (1) is the most powerful since it implicitly sees the original source of the traffic (by +virtue of being attached to that original source). It also sees the external destination of the traffic. + +Since an egress gateway will never originate its own traffic, one option is to rely on policy applied at (1) and +to allow all traffic to at (2) (either by applying no policy or by applying an "allow all"). + +Alternatively, for maximum "defense in depth" applying policy at both (1) and (2) provides extra protection should +the policy at (1) be disabled or bypassed by an attacker. Policy at (2) has the following limitations: + +- [Domain-based policy](../../network-policy/domain-based-policy.mdx) is not supported at egress from egress +gateways. It will either fail to match the expected traffic, or it will work intermittently if the egress gateway +happens to be scheduled to the same node as its clients. This is because any DNS lookup happens at the client pod. +By the time the policy reaches (2) the DNS information is lost and only the IP addresses of the traffic are available. + +- The traffic source will appear to be the egress gateway pod, the source information is lost in the address +translation that occurs inside the egress gateway pod. + +That means that policies at (2) will usually take the form of rules that match only on destination port and IP address, +either directly in the rule (via a CIDR match) or via a (non-domain based) NetworkSet. Matching on source has little +utility since the IP will always be the egress gateway and the port of translated traffic is not always preserved. + +:::note + +Since v3.15.0, $[prodname] also sends health probes to the egress gateway pods from the nodes where their clients are located. +In iptables mode, this traffic is auto-allowed at egress from the host and ingress to the egress gateway. +In eBPF mode, the probe traffic can be blocked by policy, so you must ensure that this traffic is allowed. + +::: + +## Additional resources + +Please see also: + +- [Configure egress gateways, on-premises](egress-gateway-on-prem.mdx) +- [Troubleshooting egress gateways](troubleshoot.mdx). +- [Additional configuration for egress gateway maintenance](egress-gateway-maintenance.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/egress/egress-gateway-host-ip.mdx b/calico-cloud_versioned_docs/version-24-1/networking/egress/egress-gateway-host-ip.mdx new file mode 100644 index 0000000000..a63d87fb51 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/egress/egress-gateway-host-ip.mdx @@ -0,0 +1,128 @@ +--- +description: Configure egress gateways to use the host node IP as the source address for traffic leaving the cluster. +--- + +# Configure egress gateways with host IP support + +## Big picture + +Configure an existing egress gateway deployment so that traffic exiting through it appears to come +from the **host address** of the host running the gateway pod, rather than the gateway's pod IP. + +## Value + +External firewalls and services often allowlist traffic by source IP. When egress gateway pod IPs +are not routable outside the cluster or simply not convenient to manage using the gateway's +**host address** as the source address gives external systems a stable, well-known set of IPs to +allowlist. + +Any number of application pods can multiplex their outbound traffic through a small fixed set of +egress gateways, and all of those connections will appear to come from the gateways' host address. + +:::note + +Host IP mode is most commonly used with the [on-premises setup](egress-gateway-on-prem.mdx), where +the alternative source IP would be a non-routable pod IP. On [AWS](egress-gateway-aws.mdx) and +[Azure](egress-gateway-azure.mdx) setups, gateways already use native VPC/VNet IPs that are +routable on the underlying network - enabling host IP mode there replaces those native IPs with +the host address. + +::: + +## Concepts + +### Source IP and `natOutgoing` + +When an outbound application flow leaves the cluster through an egress gateway, the source IP seen +by external services depends on the `natOutgoing` setting of the egress gateway's +[IP pool](../../reference/resources/ippool.mdx): + +- `natOutgoing: false` - the flow's source IP is the egress gateway's **pod IP**. This + is the default for all egress gateway setup guides. +- `natOutgoing: true` - the flow's source IP is the **host address** of the node where the egress + gateway pod is running. + +## Before you begin + +These instructions require a functioning egress gateway deployment. +For setup, see [our egress gateway guides](index.mdx). + +## How to + +- [Enable natOutgoing on the egress IP pool](#enable-host-ip-mode-on-the-egress-ip-pool) +- [Pin egress gateways to specific nodes](#pin-egress-gateways-to-specific-nodes) +- [Affine client pods to a specific node](#affine-client-pods-to-a-specific-node) +- [Verify the source IP](#verify-the-source-ip) + +### Enable natOutgoing on the egress IP pool + +Set `natOutgoing: true` on the IP pool used by your egress gateways: + +```bash +kubectl patch ippool egress-ippool-1 --type='merge' -p '{"spec":{"natOutgoing":true}}' +``` + +Outbound traffic leaving the cluster through a gateway in this pool will now be SNAT'd to the node +IP of the gateway's host, instead of the gateway's pod IP. + +### Pin egress gateways to specific nodes + +The source IP that external services see depends on which node the gateway pod is +scheduled to. To make this deterministic, set a `nodeSelector` on the gateway template: + +```bash +kubectl patch egressgateway egress-gateway -n default --type='merge' -p \ + '{"spec":{"template":{"spec":{"nodeSelector":{"kubernetes.io/hostname":""}}}}}' +``` + +Traffic passing through this gateway will exit the cluster with that node's IP as the +source address. + +Without pinning, the source IP will still be a node IP, but it could be any node the gateway +happens to land on. + +### Affine client pods to a specific node + +This step is optional. If you want a particular client's traffic to deterministically exit through +a particular node's IP, schedule the client to the same node as a gateway and apply an +[EgressGatewayPolicy](egress-gateway-on-prem.mdx#configure-a-namespace-or-pod-to-use-an-egress-gateway-egress-gateway-policy-method) +with `gatewayPreference: PreferNodeLocal`. The client will then prefer the gateway on its own node, +ensuring traffic exits with that node's IP. + +For example, to pin a workload to a specific node, add a `nodeSelector` to its pod spec: + +```yaml +spec: + nodeSelector: + kubernetes.io/hostname: +``` + +### Verify the source IP + +Identify the node hosting the egress gateway: + +```bash +kubectl get pod -l egress-code=red -o wide +``` + +Note the value in the `NODE` column, then look up that node's InternalIP: + +```bash +kubectl get node -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}' +``` + +Initiate an outbound connection from one of your client pods to a server outside the cluster, and +observe the source IP on the server. With host IP mode, it should match the node's InternalIP +above -- not the egress gateway's pod IP or any IP from the egress IP pool. + +:::note + +For return traffic to reach the gateway, the external server must know how to route to the egress +gateway's node IP. + +::: + +## Additional resources + +- [Egress gateway maintenance](egress-gateway-maintenance.mdx) +- [FelixConfiguration `egressIP...` fields](../../reference/resources/felixconfig.mdx#spec) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/egress/egress-gateway-maintenance.mdx b/calico-cloud_versioned_docs/version-24-1/networking/egress/egress-gateway-maintenance.mdx new file mode 100644 index 0000000000..97344c7e5b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/egress/egress-gateway-maintenance.mdx @@ -0,0 +1,337 @@ +--- +description: Reduce egress gateway downtime impact on long-lived TCP sessions in Calico Cloud connected clusters by reading termination annotations and draining gracefully. +--- + +# Optimize egress networking for workloads with long-lived TCP connections + +## Big picture + +React to egress gateway maintenance windows and minimize the impact of egress gateway downtime on sensitive workloads + +## Value + +While most workloads benefit from the reduced downtime associated with increased replicas, there are some specific cases where increasing your number of egress gateways will not have as significant an effect on availability. + +One area where this applies is when your workloads must maintain long-lived TCP connections that are coupled to higher-level abstractions, such as database sessions, transactions, or similar. In these environments, if an egress gateway becomes unavailable, these long-lived sessions may close. Data may need to be retransmitted. And in some cases, data may be lost. + +## Before you begin + +These features require you to have configured a functioning egress gateway deployment in a cluster. For more information on deploying egress gateways, [see our other egress gateway guides](index.mdx) + +## How to + +- [Observe gateway maintenance impact](#observe-gateway-maintenance-impact) +- [Expose gateway maintenance annotations to your application](#expose-gateway-maintenance-annotations-to-your-application) +- [Reduce the impact of gateway downtime](#reduce-the-impact-of-gateway-downtime) + +### Observe gateway maintenance impact + +A number of egress-related annotations are automatically added to your workloads when an egress gateway they use is in the "terminating" phase. These annotations will outline _which_ gateway is about to terminate, _when it began terminating_, and _when it will fully terminate_. This information can prove useful for conducting non-disruptive maintenance on your cluster, as it means any planned termination of egress pods will be communicated to dependent workloads. + +Before we can observe these annotations, we must first configure a _termination grace period_ for our egress gateways. The termination grace period prolongs an egress gateway's termination phase, giving us a window to react to the termination. Without configuring the grace period, we would have a zero-second window to react to gateway termination. + +#### Add a termination grace period to egress gateway replicas + +To widen our maintenance window, we can adjust the `terminationGracePeriodSeconds` field on our egress gateway pods. The amount of time we set for the termination grace period will dictate how much time a dependent workload has to prepare for a gateway going down. + +Let's add a termination grace period of 60 seconds to all pods in our egress gateway deployment, so that our egress-dependent workloads have a wider window to react: + +```sh +$ patch a termination grace period of 60s into egress gateway deployment +$ kubectl get deploy +NAME READY UP-TO-DATE AVAILABLE AGE +egress-gateway 2/2 2 2 16m + +$ patch our deployment with a 60-second termination grace period +$ kubectl patch deploy egress-gateway --type=merge -p '{"spec": {"template": {"spec": {"terminationGracePeriodSeconds": 60}}}}' +deployment.apps/egress-gateway patched + +$ wait for change to rollout +$ kubectl rollout status deploy/egress-gateway +``` + +:::note + +- Making the above alterations to an egress gateway deployment will trigger a new rollout - you can monitor the rollout status with `kubectl rollout status deploy/` +- If your rollout seems to have stalled and egress gateway pods are stuck on "ContainerCreating" phase, it's possible the deployment's IPPool has been exhausted. You can check if this is the case by inspecting a stuck pod with `kubectl describe pod ` + +::: + +#### Inspect workload annotations + +Once the updated egress gateway deployment rolls out, we're ready to observe the gateway maintenance annotations $[prodname] adds to your dependent workloads. Let's simulate cluster maintenance by deleting an egress gateway pod. It should take 60 seconds to terminate - the amount of time defined by `terminationGracePeriodSeconds`. + +```sh +$ show pods in default namespace - two egress gateway pods and one application pod using them. +$ kubectl get pods -o wide + +NAME READY STATUS RESTARTS AGE IP +application-with-long-lived-connections 1/1 Running 0 20m 192.168.192.210 +egress-gateway-6576ccdf66-fxdvh 1/1 Running 0 3m 10.10.10.1 +egress-gateway-6644fbb56b-5xbh2 1/1 Running 0 3m 10.10.10.0 + +$ delete one of the egress gateways being used by the application pod - do not block waiting for termination to finish +$ kubectl delete pod egress-gateway-6576ccdf66-fxdvh --wait=false +pod "egress-gateway-6576ccdf66-fxdvh" deleted +``` + +The gateway we just deleted should now wait in the "terminating" phase until its termination grace period expires, at-which point it will be deleted. If our application pod depends on the terminating egress gateway, we'll see gateway maintenance annotations added to the dependent application pod automatically, outlining what gateway is going down, when it began terminating, and when it will be deleted: + +```sh +$ observe the annotations added to the dependent application pod +$ kubectl get pod application-with-long-lived-connections -o yaml +apiVersion: v1 +kind: Pod +metadata: + annotations: + ... + egress.projectcalico.org/gatewayMaintenanceFinishedTimestamp: "2022-04-19T16:00:18Z" + egress.projectcalico.org/gatewayMaintenanceGatewayIP: 10.10.10.1 + egress.projectcalico.org/gatewayMaintenanceStartedTimestamp: "2022-04-19T15:59:18Z" +... +``` + +Success! Our workload's annotations mark a 60-second maintenance window for the gateway we terminated, indicating when the egress gateway began terminating, and when it will fully terminate. + +:::note + +- Adjusting egress deployments, say, by modifying the `terminationGracePeriodSeconds` field, will trigger a new rollout. +- Egress pods terminating due to a new rollout will behave the same as if they were deleted for maintenance - dependent workloads will receive gateway maintenance annotations, and gateway pods will terminate after their termination grace period has elapsed. +- Deleting an egress gateway in a way that overrides the termination grace period, say, by using `kubectl delete pod my-pod --grace-period=0`, will result in the gateway going down immediately, and dependent workloads will not have any time to react to the termination. + +::: + +
+ +### Expose gateway maintenance annotations to your application + +While the presence of gateway maintenance annotations may be useful to a cluster administrator inspecting pods, it's not quite enough if our workload wishes to react to terminating egress gateways, say, by restarting its session gracefully before loss of connectivity. + +The [Kubernetes downward API](https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/#the-downward-api) provides a means of exposing pod information [as files](https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/#store-pod-fields) or as [environment variables](https://kubernetes.io/docs/tasks/inject-data-application/environment-variable-expose-pod-information/#use-pod-fields-as-values-for-environment-variables) within the pod. This value can then be polled by your workload, to react to changes as you see fit. + +Let's write a simple pod manifest that uses the downward API to expose maintenance annotations to the program running within: + +```yaml +apiVersion: v1 +kind: Pod +metadata: + annotations: + egress.projectcalico.org/selector: egress-code == 'red' + egress.projectcalico.org/namespaceSelector: kubernetes.io/metadata.name == 'default' + name: poll-my-own-annotations + namespace: default +spec: + containers: + - name: application-container + image: k8s.gcr.io/busybox:1.24 + command: ['sh', '-c'] + args: + - while true; do + echo 'polling egress maintenance timestamp...'; + if [[ -e /var/run/egress/gatewayMaintenanceStartedTimestamp ]]; then + echo -n 'gatewayMaintenanceStartedTimestamp has value "'; cat /var/run/egress/gatewayMaintenanceStartedTimestamp; echo -en '"\n'; + fi; + sleep 3; + done; + volumeMounts: + - name: egress-maintenance-started + mountPath: /var/run/egress + volumes: + - name: egress-maintenance-started + downwardAPI: + items: + - path: 'gatewayMaintenanceStartedTimestamp' + fieldRef: + fieldPath: metadata.annotations['egress.projectcalico.org/gatewayMaintenanceStartedTimestamp'] +``` + +This sample manifest will create a pod whose `gatewayMaintenanceStartedTimestamp` annotation is mounted to the file `/var/run/egress/gatewayMaintenanceStartedTimestamp`. The pod's main process is a script which polls the value of this file. + +After deleting an egress gateway this workload relies on, let's check its logs: + +```sh +$ kubectl logs poll-my-own-annotations +polling egress maintenance timestamp... +gatewayMaintenanceStartedTimestamp has value "" +polling egress maintenance timestamp... +gatewayMaintenanceStartedTimestamp has value "" +polling egress maintenance timestamp... +gatewayMaintenanceStartedTimestamp has value "" +polling egress maintenance timestamp... +gatewayMaintenanceStartedTimestamp has value "2022-04-19T17:24:46Z" +polling egress maintenance timestamp... +gatewayMaintenanceStartedTimestamp has value "2022-04-19T17:24:46Z" +``` + +We can see above that our script saw the value of the mounted volume change at the same time what we terminated our egress gateway pod. This work can be further developed to propagate notifications to our production workloads, without any need for polling kubernetes itself. + +**Note**: It's not recommended to couple your production applications to a Kubernetes client for the purposes of polling pod information, as it could give an attacker greater privileges if successful in compromising a workload. Instead, use a method such as the downward API that fully decouples the program. + +
+ +### Reduce the impact of gateway downtime + +So far we have observed egress gateway maintenance windows, added a termination grace period to gateway pods, and propagated maintenance information directly to our workloads. Finally, we are going to look at the `maxNextHops` annotation, which is designed to limit the impact of a terminating egress gateway. + +Below is a sample kubernetes deployment which was adapted from the earlier annotation-aware pod manifest. The deployment has 3 replicas, and is configured to use egress gateways: + +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: annotation-aware-workloads +spec: + replicas: 3 + selector: + matchLabels: + app: annotation-aware-workload + strategy: + rollingUpdate: + maxSurge: 25% + maxUnavailable: 25% + type: RollingUpdate + template: + metadata: + labels: + app: annotation-aware-workload + annotations: + egress.projectcalico.org/selector: egress-code == 'red' + egress.projectcalico.org/namespaceSelector: kubernetes.io/metadata.name == 'default' + spec: + containers: + - name: application-container + image: k8s.gcr.io/busybox:1.24 + command: ['sh', '-c'] + args: + - while true; do + echo "[${MY_POD_NAME}] polling egress maintenance timestamp..."; + if [[ -e /var/run/egress/gatewayMaintenanceStartedTimestamp ]]; then + echo -n "[${MY_POD_NAME}] gatewayMaintenanceStartedTimestamp has value '"; cat /var/run/egress/gatewayMaintenanceStartedTimestamp; echo -en "'\n"; + fi; + sleep 3; + done; + volumeMounts: + - name: egress-maintenance-started + mountPath: /var/run/egress + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + volumes: + - name: egress-maintenance-started + downwardAPI: + items: + - path: 'gatewayMaintenanceStartedTimestamp' + fieldRef: + fieldPath: metadata.annotations['egress.projectcalico.org/gatewayMaintenanceStartedTimestamp'] +``` + +#### Observing the impact of egress gateway maintenance + +We now want to conduct maintenance on a particular node in our cluster, resulting in one of our egress gateways terminating. In this scenario, since our dependent applications' connections are load-balanced evenly across all available egress gateways, all of our application pods will receive a maintenance window annotation. If we have configured our applications to react to such a window, we will see them all react at once in a thundering herd: + +```sh +$ begin the termination of an egress gateway pod +$ kubectl delete pod egress-gateway-6576ccdf66-mtqzl --wait=false +pod "egress-gateway-6576ccdf66-mtqzl" deleted + +$ lets collect logs from all pods in our maintenance-aware application deployment +$ kubectl logs --selector=app=annotation-aware-workload +[annotation-aware-workloads-7987f55c9f-f7mkq] polling egress maintenance timestamp... +[annotation-aware-workloads-7987f55c9f-f7mkq] gatewayMaintenanceStartedTimestamp has value '' +[annotation-aware-workloads-7987f55c9f-qtcs2] polling egress maintenance timestamp... +[annotation-aware-workloads-7987f55c9f-qtcs2] gatewayMaintenanceStartedTimestamp has value '' +[annotation-aware-workloads-7987f55c9f-z5x25] polling egress maintenance timestamp... +[annotation-aware-workloads-7987f55c9f-z5x25] gatewayMaintenanceStartedTimestamp has value '' +[annotation-aware-workloads-7987f55c9f-qtcs2] polling egress maintenance timestamp... +[annotation-aware-workloads-7987f55c9f-qtcs2] gatewayMaintenanceStartedTimestamp has value '2022-04-20T12:24:34Z' +[annotation-aware-workloads-7987f55c9f-f7mkq] polling egress maintenance timestamp... +[annotation-aware-workloads-7987f55c9f-f7mkq] gatewayMaintenanceStartedTimestamp has value '2022-04-20T12:24:34Z' +[annotation-aware-workloads-7987f55c9f-z5x25] polling egress maintenance timestamp... +[annotation-aware-workloads-7987f55c9f-z5x25] gatewayMaintenanceStartedTimestamp has value '2022-04-20T12:24:34Z' +``` + +We can see in the above logs that all of our applications have been affected by the downtime of just a single egress gateway. In the worst case, this could lead to a window of downtime for the application, as all replicas scramble to restart their connections at once. To avoid this, lets use the `egress.projectcalico.org/maxNextHops` annotation to restrict the total number of gateways each application can depend on. + +#### Reducing the impact of egress gateway maintenance + +To place a limit on the number of egress gateways an application can depend on, annotate the application's pod with the `egress.projectcalico.org/maxNextHops` annotation. Alternatively, to limit all pods in a certain namespace, annotate that namespace. Let's annotate all pods in our sample deployment from earlier: + +```sh +kubectl patch deploy annotation-aware-workloads --type=merge -p \ +'{"spec": + {"template": + {"metadata": + {"annotations": + {"egress.projectcalico.org/maxNextHops": "1"} + } + } + } +}' +``` + +:::note + +- Either a _pod or a namespace_ can be annotated with `egress.projectcalico.org/maxNextHops`, however, the `egress.projectcalico.org/selector` annotation must also be present on the selected resource. +- If annotating pods, the `egressIPSupport` Felixconfiguration option must be set to `EnabledPerNamespaceOrPerPod`. +- If a pod's desired `maxNextHops` exceeds the total number of available egress gateways, scaling up the egress gateway deployment will result in the pod's egress networking updating until the desired number of gateways are being used. +- In all other cases, the `maxNextHops` annotation only takes effect at the time a pod is created. To ensure a pod's egress networking remains functional for its entire lifecycle, modifications to `maxNextHops` after a pod's creation will have no effect. For this reason, it's recommended that any egress gateway deployments have been scaled prior to deploying dependent workloads. + +::: + +After our patched sample deployment has been fully rolled out, each application pod should now depend on at most one egress gateway replica. Let's bring down another egress gateway pod and monitor our application logs: + +```sh +$ begin the termination of an egress gateway pod +$ kubectl delete pod egress-gateway-6576ccdf66-c42v7 --wait=false +pod "egress-gateway-6576ccdf66-c42v7" deleted + +$ collect logs from each application pod +$ kubectl logs --selector=app=annotation-aware-workload +[annotation-aware-workloads-565b6855b9-tjvqr] polling egress maintenance timestamp... +[annotation-aware-workloads-565b6855b9-tjvqr] gatewayMaintenanceStartedTimestamp has value '' +[annotation-aware-workloads-565b6855b9-s44pt] polling egress maintenance timestamp... +[annotation-aware-workloads-565b6855b9-s44pt] gatewayMaintenanceStartedTimestamp has value '' +[annotation-aware-workloads-565b6855b9-46cw5] polling egress maintenance timestamp... +[annotation-aware-workloads-565b6855b9-46cw5] gatewayMaintenanceStartedTimestamp has value '' +[annotation-aware-workloads-565b6855b9-tjvqr] polling egress maintenance timestamp... +[annotation-aware-workloads-565b6855b9-tjvqr] gatewayMaintenanceStartedTimestamp has value '2022-04-20T12:53:32Z' +[annotation-aware-workloads-565b6855b9-s44pt] polling egress maintenance timestamp... +[annotation-aware-workloads-565b6855b9-s44pt] gatewayMaintenanceStartedTimestamp has value '' +[annotation-aware-workloads-565b6855b9-46cw5] polling egress maintenance timestamp... +[annotation-aware-workloads-565b6855b9-46cw5] gatewayMaintenanceStartedTimestamp has value '' +``` + +We can see from the above logs that only a single application pod has now been affected by the terminating egress gateway. The other pods have not received an annotation for a terminating gateway because they have chosen different gateways to depend on, and thus won't be affected. + +:::note + +The subset of egress gateway replicas that each pod will depend on when using the `maxNextHops` annotation can't be manually selected. +$[prodname] selects a subset of replicas in such a way as to evenly distribute load across the whole replica set. + +::: + +
+ +## Reference + +**The following are annotations that $[prodname] sets automatically on any egress-dependent pods:** + +| Annotation | Description | Datatype | Default value | Expected values | +| -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | ---------- | ------------- | ---------------------------------------------------- | +| `egress.projectcalico.org/gatewayMaintenanceGatewayIP` | Indicates the IP of a terminating egress gateway your pod is using. | IP Address | "" | Any IP within the egress gateway deployment's IPSet. | +| `egress.projectcalico.org/gatewayMaintenanceStartedTimestamp` | Indicates when the egress gateway identified by `gatewayMaintenanceGatewayIP` began terminating. | String | "" | An RFC3339 date string | +| `egress.projectcalico.org/gatewayMaintenanceFinishedTimestamp` | Indicates when the egress gateway identified by `gatewayMaintenanceGatewayIP` will finish terminating. | String | "" | An RFC3339 date string | + +
+ +**The following annotations are used to configure your egress-dependent workloads. These annotations can be set either on a namespace, or on a pod. If setting the annotations on pods, the `egressIPSupport` FelixConfiguration option must be set to `EnabledPerNamespaceOrPerPod`.** + +| Annotation | Description | Datatype | Default value | Possible values | +| -------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------- | ------------- | ----------------------- | +| `egress.projectcalico.org/maxNextHops` | Specifies the maximum number of egress gateway replicas from the selected deployment that a pod should depend on. Replicas will be chosen in a manner that attempts to balance load across the whole egress gateway replicaset. If unset, or set to "0", egress traffic will behave in the default manner (load balanced over all available gateways). | String | "" | "0", "1", "2", "3", ... | + +## Additional resources + +- [Troubleshooting egress gateways](troubleshoot.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/networking/egress/egress-gateway-on-prem.mdx b/calico-cloud_versioned_docs/version-24-1/networking/egress/egress-gateway-on-prem.mdx new file mode 100644 index 0000000000..993ccbd497 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/egress/egress-gateway-on-prem.mdx @@ -0,0 +1,609 @@ +--- +description: Send selected application traffic through Calico Cloud egress gateways on-premises so external firewalls see a predictable source IP for cluster workloads. +redirect_from: + - /compliance/egress-gateways +--- + +# Configure egress gateways, on-premises + +## Big picture + +Configure specific application traffic to exit the cluster through an egress gateway. + +## Value + +When traffic from particular applications leaves the cluster to access an external destination, it +can be useful to control the source IP of that traffic. For example, there may be an additional +firewall around the cluster, whose purpose includes policing external accesses from the cluster, and +specifically that particular external destinations can only be accessed from authorised workloads +within the cluster. + +$[prodname]'s own policy (including [DNS policy](../../network-policy/domain-based-policy.mdx)) and +per-node firewalls can ensure this, but deployments may like to deepen their defense by adding an +external firewall as well. If the external firewall is configured to allow outbound connections +only from particular source IPs, and the intended cluster workloads can be configured so that their +outbound traffic will have one of those source IPs, then the defense in depth objective is achieved. + +$[prodname] allows specifying an [IP pool](../../reference/resources/ippool.mdx) for each pod or namespace, and +even a [specific IP](../ipam/use-specific-ip.mdx) for a new pod, but this requires predicting how many pods +there will be representing a particular application, so that the IP pool can be correctly sized. +When IPs are a precious resource, over-sizing the pool is wasteful; but under-sizing is also +problematic, as then IPs will not be available within the desired range as the application is +scaled. + +Egress gateways provide an alternative approach. Application pods and namespaces are provisioned +with IPs from the default (and presumably plentiful) pool, but also configured so that their +outbound traffic is directed through an egress gateway. (Or, for resilience, through one of a small +number of egress gateways.) The egress gateways are set up to use a [specific IP pool](../ipam/legacy-firewalls.mdx) + and to perform an SNAT on the traffic passing through them. Hence, any +number of application pods can have their outbound connections multiplexed through a fixed small +number of egress gateways, and all of those outbound connections acquire a source IP from the egress +gateway IP pool. + +:::note + +The source port of an outbound flow through an egress gateway can generally _not_ be +preserved. Changing the source port is how Linux maps flows from many upstream IPs onto a single +downstream IP. + +::: + +Egress gateways are also useful if there is a reason for wanting all outbound traffic from a +particular application to leave the cluster through a particular node or nodes. For this case, the +gateways just need to be scheduled to the desired nodes, and the application pods/namespaces +configured to use those gateways. + +## Concepts + +### Egress gateway + +An egress gateway acts as a transit pod for the outbound application traffic that is configured to +use it. As traffic leaving the cluster passes through the egress gateway, its source IP is changed +to that of the egress gateway pod, and the traffic is then forwarded on. + +### Source IP + +When an outbound application flow leaves the cluster, its IP packets will have a source IP. +Normally this is the pod IP of the pod that originated the flow, or the node IP of the node hosting +that pod. It will be the **node IP** if the pod IP came from an [IP pool](../../reference/resources/ippool.mdx) with `natOutgoing: true`, and the **pod IP** if +not. (Assuming no other CNI plugin has been configured to NAT outgoing traffic.) + +With an egress gateway involved that is all still true, except that now it's the egress gateway that +counts, instead of the original application pod. So the flow will have the egress gateway's **node +IP**, if the egress gateway's pod IP came from an [IP pool](../../reference/resources/ippool.mdx) + with `natOutgoing: true`, and the egress +gateway's **pod IP** otherwise. + +### Control the use of egress gateways + +If a cluster ascribes special meaning to traffic flowing through egress gateways, it will be +important to control when cluster users can configure their pods and namespaces to use them, so that +non-special pods cannot impersonate the special meaning. + +If namespaces in a cluster can only be provisioned by cluster admins, one option is to enable egress +gateway function only on a per-namespace basis. Then only cluster admins will be able to configure +any egress gateway usage. + +Otherwise -- if namespace provisioning is open to users in general, or if it's desirable for egress +gateway function to be enabled both per-namespace and per-pod -- a [Kubernetes admission controller](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/) + will be +needed. This is a task for each deployment to implement for itself, but possible approaches include +the following. + +1. Decide whether a given Namespace or Pod is permitted to use egress annotations at all, based on + other details of the Namespace or Pod definition. + +1. Evaluate egress annotation selectors to determine the egress gateways that they map to, and + decide whether that usage is acceptable. + +1. Impose the cluster's own bespoke scheme for a Namespace or Pod to identify the egress gateways + that it wants to use, less general than $[prodname]'s egress annotations. Then the + admission controller would police those bespoke annotations (that that cluster's users could + place on Namespace or Pod resources) and either reject the operation in hand, or allow it + through after adding the corresponding $[prodname] egress annotations. + +### Policy enforcement for flows via an egress gateway + +For an outbound connection from a client pod, via an egress gateway, to a destination outside the +cluster, there is more than one possible enforcement point for policy: + +The path of the traffic through policy is as follows: + +1. Packet leaves the client pod and passes through its egress policy. +2. The packet is encapsulated by the client pod's host and sent to the egress gateway +3. The encapsulated packet is sent from the host to the egress gateway pod. +4. The egress gateway pod de-encapsulates the packet and sends the packet out again with its own address. +5. The packet leaves the egress gateway pod through its egress policy. + +To ensure correct operation, (as of v3.15) the encapsulated traffic between host and egress gateway is auto-allowed by +$[prodname] and other ingress traffic is blocked. That means that there are effectively two places where +policy can be applied: + +1. on egress from the client pod +2. on egress from the egress gateway pod (see limitations below). + +The policy applied at (1) is the most powerful since it implicitly sees the original source of the traffic (by +virtue of being attached to that original source). It also sees the external destination of the traffic. + +Since an egress gateway will never originate its own traffic, one option is to rely on policy applied at (1) and +to allow all traffic to at (2) (either by applying no policy or by applying an "allow all"). + +Alternatively, for maximum "defense in depth" applying policy at both (1) and (2) provides extra protection should +the policy at (1) be disabled or bypassed by an attacker. Policy at (2) has the following limitations: + +- [Domain-based policy](../../network-policy/domain-based-policy.mdx) is not supported at egress from egress + gateways. It will either fail to match the expected traffic, or it will work intermittently if the egress gateway + happens to be scheduled to the same node as its clients. This is because any DNS lookup happens at the client pod. + By the time the policy reaches (2) the DNS information is lost and only the IP addresses of the traffic are available. + +- The traffic source will appear to be the egress gateway pod, the source information is lost in the address + translation that occurs inside the egress gateway pod. + +That means that policies at (2) will usually take the form of rules that match only on destination port and IP address, +either directly in the rule (via a CIDR match) or via a (non-domain based) NetworkSet. Matching on source has little +utility since the IP will always be the egress gateway and the port of translated traffic is not always preserved. + +:::note + +Since v3.15.0, $[prodname] also sends health probes to the egress gateway pods from the nodes where +their clients are located. In iptables mode, this traffic is auto-allowed at egress from the host and ingress +to the egress gateway. In eBPF mode, the probe traffic can be blocked by policy, so you must ensure that this traffic allowed; this should be fixed in an upcoming +patch release. + +::: + +## Before you begin + +**Unsupported** + +- GKE + +**Required** + +- Calico CNI +- Open port UDP 4790 on the host + +## How to + +- [Enable egress gateway support](#enable-egress-gateway-support) +- [Provision an egress IP pool](#provision-an-egress-ip-pool) +- [Deploy a group of egress gateways](#deploy-a-group-of-egress-gateways) +- [Configure iptables backend for egress gateways](#configure-iptables-backend-for-egress-gateways) +- [Configure namespaces and pods to use egress gateways](#configure-namespaces-and-pods-to-use-egress-gateways) +- [Optionally enable ECMP load balancing](#optionally-enable-ecmp-load-balancing) +- [Verify the feature operation](#verify-the-feature-operation) +- [Control the use of egress gateways](#control-the-use-of-egress-gateways) +- [Upgrade egress gateways](#upgrade-egress-gateways) + +### Enable egress gateway support + +In the default **FelixConfiguration**, set the `egressIPSupport` field to `EnabledPerNamespace` or +`EnabledPerNamespaceOrPerPod`, according to the level of support that you need in your cluster. For +support on a per-namespace basis only: + +```bash +kubectl patch felixconfiguration default --type='merge' -p \ + '{"spec":{"egressIPSupport":"EnabledPerNamespace"}}' +``` + +Or for support both per-namespace and per-pod: + +```bash +kubectl patch felixconfiguration default --type='merge' -p \ + '{"spec":{"egressIPSupport":"EnabledPerNamespaceOrPerPod"}}' +``` + +:::note + +- `egressIPSupport` must be the same on all cluster nodes, so you should set them only in the + `default` FelixConfiguration resource. +- The operator automatically enables the required policy sync API in the FelixConfiguration. + +::: + +### Provision an egress IP pool + +Provision a small IP Pool with the range of source IPs that you want to use for a particular +application when it connects to an external service. For example: + +```bash +kubectl apply -f - < egress.projectcalico.org/selector="egress-code == 'red'" +``` + +By default, that selector can only match egress gateways in the same namespace. To select gateways +in a different namespace, specify a `namespaceSelector` annotation as well, like this: + +```bash +kubectl annotate ns egress.projectcalico.org/namespaceSelector="kubernetes.io/metadata.name == 'default'" +``` + +Egress gateway annotations have the same [syntax and range of expressions](../../reference/resources/networkpolicy.mdx#selector) as the selector fields in +$[prodname] [network policy](../../reference/resources/networkpolicy.mdx#entityrule). + +To configure a specific Kubernetes Pod to use egress gateways, specify the same annotations when +creating the pod. For example: + +```bash +kubectl apply -f - < egress.projectcalico.org/egressGatewayPolicy="egw-policy1" +``` + +To configure a specific Kubernetes pod to use the same policy, specify the same annotations when +creating the pod. +For example: + +```bash +kubectl apply -f - < -n -- nc 8089 ` should be the IP address of the netcat server. + +Then, if you check the logs or output of the netcat server, you should see: + +``` +Connection from received +``` + +with `` being one of the IPs of the egress IP pool that you provisioned. + +## Upgrade egress gateways + +From v3.16, egress gateway deployments are managed by the Tigera Operator. + +- When upgrading from a pre-v3.16 release, no automatic upgrade will occur. To upgrade a pre-v3.16 egress gateway deployment, + create an equivalent EgressGateway resource with the same namespace and the same name as mentioned [above](#deploy-a-group-of-egress-gateways); + the operator will then take over management of the old Deployment resource, replacing it with the upgraded version. + +- Use `kubectl apply` to create the egress gateway resource. Tigera Operator will read the newly created resource and wait + for the other $[prodname] components to be upgraded. Once the other $[prodname] components are upgraded, Tigera Operator + will upgrade the existing egress gateway deployment with the new image. + +By default, upgrading egress gateways will sever any connections that are flowing through them. To minimise impact, +the egress gateway feature supports some advanced options that give feedback to affected pods. For more details see +the [egress gateway maintenance guide](egress-gateway-maintenance.mdx). + +## Next steps + +Make your egress gateway reachable from outside the cluster. An egress gateway's source IP is a pod IP, so external hosts and firewalls can reach it — and return traffic can find its way back — only when that IP is routable beyond the node. There are two ways to do this: + +- [Advertise it over BGP](../configuring/bgp.mdx) — once you peer with your network infrastructure, $[prodname] exports egress gateway IPs to the peered routers automatically. +- [Make it reachable on the local L2 segment without BGP](../configuring/local-subnet-l2-reachability.mdx) — give egress gateways IPs from a no-encapsulation IP pool within the host subnet and let $[prodname] answer ARP/NDP for them. Use this in flat L2 or on-premises environments where BGP is not an option. + +## Additional resources + +Please see also: + +- The `egressIP...` fields of the [FelixConfiguration resource](../../reference/resources/felixconfig.mdx#spec). +- [Additional configuration for egress gateway maintenance](egress-gateway-maintenance.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/egress/external-network.mdx b/calico-cloud_versioned_docs/version-24-1/networking/egress/external-network.mdx new file mode 100644 index 0000000000..26f1014c5d --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/egress/external-network.mdx @@ -0,0 +1,182 @@ +--- +description: Direct Calico Cloud egress gateway traffic onto multiple external networks with potentially overlapping IPs by associating gateways with named ExternalNetworks. +--- + +# Configure egress traffic to multiple external networks + +## Big picture + +Allows workloads from different namespaces of a Kubernetes cluster to egress onto different external networks that (may) have overlapping IPs with each other. + +## Value + +When traffic from particular applications leaves the cluster to access an external destination, egress gateway enables users to control the source IP of that traffic. However, in this case, egress gateway sees all the external destinations as a flat L3 network. If a user has a network topology which involves multiple external networks and the service endpoints in those networks may have overlapping IPs, it becomes a mandatory requirement that $[prodname] should be able to direct the egress traffic to the service endpoint in the correct network. + +For example, suppose a user has a network setup as below: +There are two external networks (Red and Yellow) outside the cluster. A web server on network Red is exposing its service at `105.8.10.1`. A database server on network Yellow is exposing its service at `105.8.10.1` too. + +Then the cluster could have requirements as follows: +Pods in Red namespace should send traffic via egress gateways to `105.8.10.1` on external network Red. +Pods in Yellow namespace should send traffic via egress gateways to `105.8.10.1` on external network Yellow. + +External network support is introduced into $[prodname] to meet those requirements. It allows user to associate a egress gateway to external networks so that the egress traffic originated from the client via that egress gateway will be routed to the service endpoints on the associated external networks. In a nutshell, the external network feature adds support for L3 segmentation of outgoing egress traffic. + +## Prerequisites + +- IPs on multiple external networks may overlap with each other, however, those IPs must not overlap with the IPs of pods within the cluster or their hosts. +- IPs of the BGP peers peering with the cluster should not overlap. + +## Limitations + +- This feature should be used in combination with egress gateways. Sending traffic from a client pod to external networks without going through an egress gateway is not supported. +- This feature only supports IPv4. + +## Before you begin + +This feature requires you to understand how to configure a functioning egress gateway deployment in a cluster. For more information on deploying egress gateways, [see our other egress gateway guides](./). + +## Concepts + +### Egress node +A node in the cluster which is peering with external networks and populating routes to direct traffic from the egress gateway to the destinations on external networks. Any egress gateway which is associated to an external network should be scheduled to be running on a egress node. + +### BGP bootstrap routes +BGP bootstrapping routes are routes on the node for traffic from that node to reach other nodes in the cluster, or other destinations outside the cluster, via whatever routers and external networks the user may need. This is considered outside of the scope of $[prodname], as both $[prodname] and external BGP routers on external networks may require bootstrapping routes to be set up before peering with each other. + +## How to + +### Enable egress gateway support + +In the default **FelixConfiguration**, set the `egressIPSupport` field to `EnabledPerNamespace` or +`EnabledPerNamespaceOrPerPod`, according to the level of granularity that you need in your cluster. For support on a per-namespace basis only: + +```bash +kubectl patch felixconfiguration default --type='merge' -p \ + '{"spec":{"egressIPSupport":"EnabledPerNamespace"}}' +``` + +Or to support both per-namespace and per-pod: + +```bash +kubectl patch felixconfiguration default --type='merge' -p \ + '{"spec":{"egressIPSupport":"EnabledPerNamespaceOrPerPod"}}' +``` + +### Enable external network support + +In the default **FelixConfiguration**, set the `externalNetworkSupport` field to `Enabled`. + +```bash +kubectl patch felixconfiguration default --type='merge' -p \ + '{"spec":{"externalNetworkSupport":"Enabled"}}' +``` + +### Create external network resources + +Add an `ExternalNetwork` resource to the cluster for each external networks. For example, to create an `ExternalNetwork` for `red` network. + +```bash +kubectl create -f - < +# - +# timeoutSeconds: 15 +# intervalSeconds: 5 +# httpProbe: +# urls: +# - +# - +# timeoutSeconds: 30 +# intervalSeconds: 10 + template: + metadata: + labels: + egress-code: red + spec: + nodeSelector: + kubernetes.io/os: linux + terminationGracePeriodSeconds: 0 +EOF +``` + +Where: +- Where externalNetworks associates this egress gateway deployments with external network `red`. All traffic leaving the egress gateway to an external destination will be routed to the destination on external network `red`. +- Node selector `egress: true` forces egress gateway pods of the deployment to be scheduled on egress nodes. + +:::warning + +If multiple networks in the `externalNetworkNames` list (of the egress gateway) advertise the same prefix to $[prodname] then traffic to those prefixes will flow to a non-deterministic network, which may result in hard-to-debug connectivity issues. It is the user's responsibility to make sure it will never happen. + +::: + +### Use egress gateways + +Add annotations to namespaces or pods to configure them to use the relevant egress gateways created by the step above. Egress traffic from the application pods will be routed to destinations within the designated external networks. + +## Additional resources + +- [Troubleshooting egress gateways](troubleshoot.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/networking/egress/index.mdx b/calico-cloud_versioned_docs/version-24-1/networking/egress/index.mdx new file mode 100644 index 0000000000..2f6f3eae67 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/egress/index.mdx @@ -0,0 +1,11 @@ +--- +description: Egress gateways in Calico Cloud — pin per-namespace or per-pod source IPs for outbound traffic, segment external networks, and integrate with cloud fabrics. +hide_table_of_contents: true +--- + +# Egress gateways + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/networking/egress/troubleshoot.mdx b/calico-cloud_versioned_docs/version-24-1/networking/egress/troubleshoot.mdx new file mode 100644 index 0000000000..3f2ecd7a8c --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/egress/troubleshoot.mdx @@ -0,0 +1,276 @@ +--- +description: Troubleshooting guide for Calico Cloud egress gateways covering connection failures, source IP mismatches, BGP route propagation, and required pod metadata. +--- + +# Troubleshoot egress gateways + +- [Checklist of common problems](#checklist-of-common-problems) +- [Connection to an egress gateway cannot be established](#connection-to-an-egress-gateway-cannot-be-established) +- [Connection to an egress gateway is established, but destination is not getting correct IP](#connection-to-an-egress-gateway-is-established-but-destination-is-not-getting-correct-ip) + +## Checklist of common problems + +Use the following checklist to troubleshoot, or to collect details before opening a Support ticket. + +### Is the egress gateway feature enabled? + +Egress gateway is disabled by default.  Have you enabled it in [Felix configuration](../../reference/resources/felixconfig.mdx) by setting `egressIPSupport` to `EnabledPerNamespace` or `EnabledPerNamespaceOrPerPod`? + +### Does your egress gateway routing go through a router? (On-prem only.) + +As shown in the following diagram, from the gateway to the destination, the source IP is the egress IP.  On the return path, from the destination back to the gateway, the destination IP is the egress IP. If there are any routers between the gateway and the destination, they must all know how to route the egress IP back to the gateway. If they don’t, the attempted connection cannot be established. + +![egress-basic-routing](/img/calico-enterprise/egress-basic-routing.svg) + +Options to make routers aware of egress IP: + +- Program routes statically on routers +- Peer routers with the cluster, directly or indirectly using BGP or some other protocol, or other method so routers learn about the egress IP + +### Does your egress gateway have required metadata? + +Review important egress gateway metadata (for example, namespace and labels); they are required for a client to identify the gateway(s) that it should use. + +### Is natOutgoing on your IPPool set up correctly? + +For most egress gateway scenarios you should have: `natOutgoing: false` on the egress IPPool. If you have `natOutgoing: true`, the egress gateway will SNAT to its own IP, which is the intended egress gateway IP. But then the egress gateway’s node will also SNAT to its own IP (i.e. the node IP), which immediately overrides the egress gateway IP. + +### Do clients and nodes have required selectors? + +Review the following annotations that are required for the client to identify its egress gateways: + +- egress.projectcalico.org/egressGatewayPolicy +- egress.projectcalico.org/selector +- egress.projectcalico.org/namespaceSelector + +on + +- Client pod +- Client pod’s namespace + +### Does Calico have enough route tables? + +On Linux, Calico claims a number of routing tables for various purposes. The exact number of tables allocated is determined by the [FelixConfiguration](../../reference/resources/felixconfig.mdx) option `routeTableRanges`. Failing to allocate enough route tables will result in Calico failing to program the networking needed for Egress Gateways to function. + +Under these conditions, `calico-node` pods will log error messages and report [unhealthy](#check-calico-node-health). + +[See more about routeTableRanges](../../reference/resources/felixconfig.mdx#routetableranges) + +### Check egress gateway health + +As of v3.15.0, the egress gateway `Deployment` includes a Kubernetes `readinessProbe`. The egress gateway will +only report ready if it is able to connect to the local `calico-node` pod and if any configured HTTP/ICMP probes +are succeeding. + +If the egress gateway reports not-Ready then more information can be found in its logs. The egress gateway logs to +`stdout` so its logs are available via `kubectl logs -n `. + +### Check health of calico-node to egress gateway probes + +As of v3.15.0, each `calico-node` pod probes the health of the remote egress gateways that its local pods are using. +If probes fail, the failures are logged in `calico-node`'s log (search for `egress_gw_tracker`) and reported via Prometheus metrics: + +``` +felix_egress_gateway_remote_polls{status="probe-failed"} 0 +felix_egress_gateway_remote_polls{status="total"} 2 +felix_egress_gateway_remote_polls{status="up"} 2 +``` + +Where, the `total` metric reports the total number of remote egress gateways that are being polled and the `up` and `probe-failed` +metrics report the number of egress gateways in each of those states. + +### Check calico-node health + +Check that your calico-node pods are consistently running and ready, especially on the nodes hosting the client and +gateway pods. Confirming healthy pods will rule out possible issues. If you find that `calico-node` is not ready, +describing the pod should show which health check is failing: + +```bash +kubectl describe pod ... +``` + +In AWS, issues such as permissions problems will report a problem with the `"aws-eni-addresses-in-sync"` health check. +For more information on the problem, review the `calico-node` log. For example, a permissions issue will result in a +log such as the following: + +``` +2021-11-16 13:11:59.292 [ERROR][26606] felix/aws_iface_provisioner.go 343: Failed to resync with AWS. Will retry after backoff. error=failed to create ENI: operation error EC2: CreateNetworkInterface, https response error StatusCode: 403, RequestID: 13dead98-7da0-4695-9be8-80cab4d5528e, api error UnauthorizedOperation: You are not authorized to perform this operation. Encoded authorization failure message: j4x3cFwZdJ......ShGkw +``` + +If you see such a log, check the AWS IAM permissions assigned to the nodes in your cluster to ensure that the nodes +have the [required permissions](egress-gateway-aws.mdx#configure-aws-iam-roles). It is also possible +to decode the "encoded authorization failure message" in the log by following [this guide](https://aws.amazon.com/premiumsupport/knowledge-center/aws-backup-encoded-authorization-failure/); this gives more detail on the error. + +### Check IP rule and routing setup on the client node + +**Run `ip rule`** + +On the client node, run: + +``` +ip rule +``` + +**Sample output** + +You will see a line for each pod on the node that is configured to use an egress gateway. + +``` +from 192.168.24.35 fwmark 0x80000/0x80000 lookup 250 +``` + +Where: + +- `192.168.24.35` is the relevant client's pod IP +- `250` is the routing table number +- `fwmark 0x80000/0x80000` is the bit/mask + +If you don’t see this, it means one of the following: + +- egressIPSupport is not enabled +- egressIPSupport is enabled, but you have not configured egress annotations on the client pod or on its namespace +- egressIPSupport is EnabledPerNamespace and you have configured egress annotations on the client pod, but not on its namespace + +**Run `ip route show table`** + +On the client node, run the following command using the routing table number from the `ip rule` command. For example: `250`. + +``` +ip route show table +``` + +**Sample output: clients using a single egress gateway** + +``` +default via 11.11.11.1 dev egress.calico onlink +``` + +**Sample: clients using multiple gateways** + +``` +default onlink + nexthop via 11.11.11.1 dev egress.calico weight 1 onlink + nexthop via 11.11.11.2 dev egress.calico weight 1 onlink +``` + +**Sample: clients using the following egress gateway policy** + +```yaml noValidation +apiVersion: projectcalico.org/v3 +kind: EgressGatewayPolicy +metadata: + name: "egw-policy1" +spec: + rules: + - destination: + cidr: 10.0.0.0/8 + description: "Local: no gateway" + - destination: + cidr: 11.0.0.0/8 + description: "Gateway to on prem" + gateway: + namespaceSelector: "kubernetes.io/metadata.name == 'default'" + selector: "egress-code == 'blue'" + maxNextHops: 2 + - description: "Gateway to internet" + gateway: + namespaceSelector: "kubernetes.io/metadata.name == 'default'" + selector: "egress-code == 'red'" +``` + +``` +default onlink + nexthop via 11.11.11.1 dev egress.calico weight 1 onlink + nexthop via 11.11.11.2 dev egress.calico weight 1 onlink + nexthop via 11.11.11.3 dev egress.calico weight 1 onlink +throw 10.0.0.0/8 +11.0.0.0/8 onlink + nexthop via 11.11.12.1 dev egress.calico weight 1 onlink + nexthop via 11.11.12.2 dev egress.calico weight 1 onlink +``` + +If you see nothing at all, or the following: + +``` +unreachable default scope link +``` + +- Verify that you have provisioned the gateways +- Review the selectors, and gateway namespace and labels to determine why they aren’t matching each other + +### Do you have egress IPs in BGPConfiguration svcExternalIPs? + +You should not have any egress IPs or pod IP ranges in BGPConfiguration `serviceExternalIPs` or `serviceClusterIPs` fields; it causes problems if you do. + +By default, $[prodname] BGP exports all pod IPs, which includes egress gateway IPs because they are pod IPs. But you can also use [BGPConfiguration resource parameters](../../reference/resources/bgpconfig.mdx) like `BGPConfiguration`, `serviceClusterIPs`, `serviceExternalIPs` and `serviceLoadBalancerIPs` to export additional IP ranges, in particular Kubernetes Service IPs. Because $[prodname] exports additional IP ranges in a different way from pod IPs, things can go wrong if you include pod IPs in the additional ranges. + +## Connection to an egress gateway cannot be established + +If the outbound connection cannot be established, the policy may be denying the flow. As shown in the following diagram, policy is enforced at more points in an egress gateway flow. + +![egress-basic-routing](/img/calico-enterprise/egress-basic-routing.svg) + +**Policy enforcement**: + +- From the client pod, egress +- To the gateway pod, ingress +- From the gateway pod, egress +- Any relevant HostEndpoints that are configured in your cluster + +In [the web console](../../tutorials/calico-cloud-features/tour.mdx), check for dropped packets because of policy on the outbound connection path. If you are using the iptables data plane, you can also run the following command on the client and gateway nodes to look at a lower level. + +``` +watch iptables-save -c | grep DROP | grep -v 0:0 +``` + +## Connection to an egress gateway is established, but destination is not getting correct IP + +If you see that the outbound connection established, but the source IP is incorrect at the destination, this can indicate that other SNAT or MASQUERADE is taking effect after the packet leaves the egress gateway pod and is overriding the egress gateway IP. If you intentionally have a MASQUERADE/SNAT for another general purpose, you must filter it so it does not apply to traffic whose source IP comes from the egress gateway pool. + +To check the egress gateway’s node, use iptables: + +``` +iptables-save -c | grep -i MASQUERADE +iptables-save -c | grep -i SNAT +``` + +## Finding leaked AWS ENIs + +In normal usage of the AWS-backed IP pools feature, the $[noderunning] Pod on each node will manage the +secondary ENIs used for networking AWS-backed IP pools. It also marks its secondary ENIs for deletion on +instance termination to avoid leaking any ENIs when an instance is terminated. + +However, in certain highly unusual situations, such as the following sequence of events: + +- $[noderunning] adds an ENI. +- The AWS API call to mark the ENI for "delete on termination" fails. +- The entire instance is deleted before the automatic retry of the above operation succeeds. + +Then, it would be possible for an ENI to be leaked. $[prodname] marks all the ENIs that it creates with tags +to identify them as $[prodname] secondary ENIs and the ID of the instance they _should_ belong to. +To find potentially leaked ENIs, you can use the AWS command line tool as follows: + +```bash +aws ec2 describe-network-interfaces --filters Name=status,Values=available Name=tag-key,Values=calico:use +``` + +Then, examine the "Tag set" of the returned network interface values to see if any of them belong to nodes that have +been deleted: + +``` +"TagSet": [ + { + "Key": "calico:use", + "Value": "secondary" + }, + { + "Key": "calico:instance", + "Value": "i-00122bf604c6ab776" + } +], +``` + +If the instance ID recorded in the "calico:instance" tag is for an instance that no longer exists then the ENI +has been leaked; it is safe to delete the ENI. If the instance ID belongs to an active instance then there +is no need to delete the ENI, it should be cleaned up (or put into use) by the $[noderunning] Pod running +on that instance. diff --git a/calico-cloud_versioned_docs/version-24-1/networking/index.mdx b/calico-cloud_versioned_docs/version-24-1/networking/index.mdx new file mode 100644 index 0000000000..6bf6154242 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/index.mdx @@ -0,0 +1,59 @@ +--- +description: Calico Cloud networking covers BGP, overlay encapsulation, IPAM, egress gateways, ingress gateways, and per-cluster training across connected clusters. +--- + +import { DocCardLink, DocCardLinkLayout } from '/src/___new___/components'; + +# Networking + +The $[prodname] network plugins provide a range of networking options to fit your implementation and maximize performance. + +## Getting started + + + + + + +## Configuring networking + + + + + + + + + + + + + + +## IP address management + + + + + + + + + + + + + + + + +## Egress gateways + + + + + + + + + \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/networking/ingress-gateway/about-calico-ingress-gateway.mdx b/calico-cloud_versioned_docs/version-24-1/networking/ingress-gateway/about-calico-ingress-gateway.mdx new file mode 100644 index 0000000000..27e67407ee --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/ingress-gateway/about-calico-ingress-gateway.mdx @@ -0,0 +1,104 @@ +--- +description: Overview of Calico Ingress Gateway in Calico Cloud — a hardened Envoy Gateway build that brings the Kubernetes Gateway API to your connected clusters. +title: Calico Ingress Gateway +--- + +# Calico Ingress Gateway + +Calico Ingress Gateway provides a streamlined solution for managing cluster ingress traffic by using the Kubernetes Gateway API standard. + +## About Calico Ingress Gateway + +Calico Ingress Gateway is a hardened distribution of the open-source Envoy Gateway project. +Tigera rebuilds the Envoy Gateway code using hardened base images for robust security and stable performance. + +Calico Ingress Gateway provides seamless, deep integration with the Calico environment. +This approach ensures that ingress security and networking configurations are unified within the Calico framework. + +## About Gateway API + +The Kubernetes Gateway API is the modern, standardized interface for governing access to services within a cluster, serving as the successor to the original Ingress resource. + +Key features and benefits of the Gateway API include: + +* Role-oriented design: It clearly separates network infrastructure concerns (managed by network teams via Gateway resources) from application-specific routing (managed by developers via routing resources). +* Advanced configuration: The API natively supports advanced traffic control patterns, such as weighted traffic splitting for canary deployments, header manipulation, and detailed request matching, functionality that often required vendor-specific annotations in the past. +* Portability: As a standard Kubernetes API, configurations are not tied to a single vendor implementation, providing portability and preventing vendor lock-in. + +## About Envoy Gateway + +Gateway API defines the specification, and Envoy Gateway is the primary implementation framework. + +Envoy Gateway acts as the control plane that translates Gateway API resources into actionable configuration for its data plane. +The data plane utilizes Envoy Proxy, a high-performance, widely adopted proxy used across the cloud-native ecosystem for edge and service mesh functionality. +By leveraging Envoy, the Calico Ingress Gateway provides proven reliability, high throughput, and granular observability. + +## Deployment and management with Calico Ingress Gateway + +While the underlying architecture involves multiple layers (Gateway API specification, Envoy Gateway control plane, Envoy Proxy data plane), the Calico Ingress Gateway simplifies deployment and management significantly. + +Calico Ingress Gateway is managed entirely by the Tigera Operator. +This integration automates the lifecycle of the components to reduce manual configuration overhead. +CVEs are addressed as part of the regular $[prodname] patch release cadence. +Administrators provision the gateway environment simply by defining a standard Gateway resource. + +### Calico Enterprise and Calico Cloud integration + +For commercial deployments, Calico Ingress Gateway extends functionality by integrating directly with the Web Application Firewall (WAF). +This feature allows operators to secure their ingress points against the OWASP Top 10 and other common vulnerabilities through simple configuration within the Calico management plane. + +## Where Calico Ingress Gateway runs + +Calico Ingress Gateway uses a namespaced deployment model. +The Tigera Operator runs a single Envoy Gateway controller in the `calico-system` namespace. +When you create a `Gateway` resource, its proxy, `Service`, and load balancer are created in the **same namespace as the `Gateway`**. + +This model lets the owner of a namespace apply their own network policy and RBAC to the gateway's proxy pods. +Teams do not have to share a namespace, and they do not have to ask a cluster administrator to change a shared namespace on their behalf. +You can deploy a gateway in each namespace that needs one, so multiple teams each run and own a gateway in their own namespace. + +The operator also creates supporting resources in each namespace that holds a `Gateway`: + +- A `tigera-ca-bundle` ConfigMap, which the proxy mounts so that it trusts cluster and public certificate authorities. + The proxy does not start until this ConfigMap is present. +- A copy of the `tigera-pull-secret`, so the namespace can pull the hardened gateway images. +- A `waf-http-filter` ServiceAccount and RoleBinding, used by the Web Application Firewall and Layer 7 log collector sidecars. + +Each `Gateway` gets its own load balancer. +Calico Ingress Gateway does not place more than one gateway behind a single load balancer. + +## Supported Gateway API resources + +Calico Ingress Gateway supports the following Gateway API resources. + +| Resource | Versions | +| ---------------- | ---------------------- | +| BackendTLSPolicy | v1, v1alpha3 | +| GatewayClass | v1, v1beta1 | +| Gateway | v1, v1beta1 | +| GRPCRoute | v1 | +| HTTPRoute | v1, v1beta1 | +| ListenerSet | v1 | +| ReferenceGrant | v1, v1beta1 | +| TCPRoute | v1alpha2 | +| TLSRoute | v1, v1alpha2, v1alpha3 | +| UDPRoute | v1alpha2 | + +:::note + +OpenShift 4.19 introduced [restrictions](https://docs.redhat.com/en/documentation/openshift_container_platform/4.19/html-single/ingress_and_load_balancing/index#nw-ingress-gateway-api-overview_ingress-gateway-api) on which Gateway API CRDs can be installed. + +- The available Gateway API resources in OpenShift 4.19 are: `GatewayClass`, `Gateway`, `GRPCRoute`, `HTTPRoute` and `ReferenceGrant`. + +- Other Gateway API resources, namely `BackendTLSPolicy`, `ListenerSet`, `TCPRoute`, `TLSRoute` and `UDPRoute`, are not available in OpenShift 4.19. + +::: + +## Next steps + +* Learn more about Calico Ingress Gateway with the [canary deployment tutorial](tutorial-ingress-gateway-canary.mdx). + +## Additional resources + +* [Kubernetes Gateway API documentation](https://gateway-api.sigs.k8s.io/) +* [Envoy Gateway documentation](https://gateway.envoyproxy.io/docs/) \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/networking/ingress-gateway/create-ingress-gateway.mdx b/calico-cloud_versioned_docs/version-24-1/networking/ingress-gateway/create-ingress-gateway.mdx new file mode 100644 index 0000000000..0526766691 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/ingress-gateway/create-ingress-gateway.mdx @@ -0,0 +1,142 @@ +--- +description: Deploy a Calico Ingress Gateway in a Calico Cloud connected cluster by applying GatewayAPI and Gateway resources tied to the Tigera-managed gateway class. +--- + +# Create an ingress gateway + +Create an ingress gateway to manage ingress traffic with the Kubernetes Gateway API. + +## Creating an ingress gateway with Calico Ingress Gateway + +Calico Ingress Gateway is based on Envoy Gateway, an open-source implementation of the Gateway API. +Calico provides a simple, convenient way to deploy and manage this implementation. + +Creating an ingress gateway with Calico Ingress Gateway involves only a few steps. +You need to do the following: +* Enable Calico Ingress Gateway by creating a `GatewayAPI` resource. +* Create a `Gateway` resource that references `tigera-gateway-class`, the name of the `GatewayClass` resource created by the Tigera Operator. +* When you create your gateway routing resources, you refer to this `Gateway` resource. + +## Limitations + +* Calico Ingress Gateway is not supported on Windows nodes. + +## Prerequisites + +* You have an understanding of the [Kubernetes Gateway API](https://gateway-api.sigs.k8s.io/) and know how you want to configure it for your system. +* $[prodname] was installed on your cluster using the Tigera Operator. +* Your cluster is provisioned to support services of type `LoadBalancer`. + +## Create an ingress gateway + +1. To enable Gateway API support, create a `GatewayAPI` resource with the name `tigera-secure`: + + ```bash + kubectl apply -f - < + namespace: + // highlight-end + spec: + // highlight-next-line + gatewayClassName: tigera-gateway-class + ... + ``` + Replace `` with a name for your gateway, and `` with the namespace where you want the gateway to run. + You will refer to this gateway name for all services you want to use the gateway. + + The proxy, its `Service`, and its load balancer are created in the same namespace as this `Gateway`. + To check that the proxy is running, list the pods in that namespace: + + ```bash + kubectl get pods -n + ``` + +1. If your cluster enforces a [global default deny policy](../../network-policy/beginners/kubernetes-default-deny.mdx), apply a network policy in the `Gateway` namespace that lets the proxy pods reach the gateway controller, the Kubernetes API server, and DNS. + Without it, the proxy cannot start. + + ```yaml + apiVersion: projectcalico.org/v3 + kind: NetworkPolicy + metadata: + name: allow-tigera-gateway-proxy + namespace: + spec: + selector: 'app.kubernetes.io/name == "envoy"' + types: [Ingress, Egress] + ingress: + - action: Allow + egress: + - action: Allow # DNS + protocol: UDP + destination: + namespaceSelector: 'projectcalico.org/name == "kube-system"' + selector: 'k8s-app == "kube-dns"' + ports: [53] + - action: Allow # configuration from the gateway controller + protocol: TCP + destination: + namespaceSelector: 'projectcalico.org/name == "calico-system"' + selector: 'app.kubernetes.io/name == "gateway-helm"' + ports: [18000] + - action: Allow # Kubernetes API server + protocol: TCP + destination: + services: + name: kubernetes + namespace: default + ``` + Replace `` with the namespace where you created the `Gateway`. + Add egress rules for your own backends as needed. + +1. Create a gateway routing resource that refers to your `Gateway` resource as `.spec.parentRefs`: + + ```yaml title='Example snippet of an HTTPRoute resource' + apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + name: example-httproute + spec: + // highlight-start + parentRefs: + - name: + // highlight-end + ... + ``` + Replace `` with the name of your `Gateway` resource. + +## Additional resources + +* [Kubernetes Gateway API documentation](https://gateway-api.sigs.k8s.io/) +* [Envoy Gateway documentation](https://gateway.envoyproxy.io/docs/) +* [Multi-tenant WAF setup](../../threat/gateway-waf/multi-tenant-setup.mdx) +* [Tutorial: Launch a canary deployment](tutorial-ingress-gateway-canary.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/ingress-gateway/customize-ingress-gateway.mdx b/calico-cloud_versioned_docs/version-24-1/networking/ingress-gateway/customize-ingress-gateway.mdx new file mode 100644 index 0000000000..8a11c134b2 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/ingress-gateway/customize-ingress-gateway.mdx @@ -0,0 +1,252 @@ +--- +description: Tune the Calico Ingress Gateway in Calico Cloud via the GatewayAPI resource — multiple gateway classes, pod placement, container resources, and load balancer annotations. +--- + +# Customizing your ingress gateway + +You can specify core configuration elements of your ingress gateway by specifying a configuration in the `GatewayAPI` resource. +Alternatively, for the complete set of customization options available to Envoy Gateway, you can provide your own Envoy Gateway configuration. + +## Customizing an ingress gateway with the `GatewayAPI` resource + +Many customizations are available for the `GatewayAPI` resource. +This resource has fields that allow some aspects of Gateway deployments to be customized. +For example: +- `spec.gatewayControllerDeployment.spec.template.spec.nodeSelector` allows control over where the gateway controller is scheduled. +- `spec.gatewayClasses` allows the provisioning of multiple `GatewayClass` resources, each with its own set of customizations, instead of the single `tigera-gateway-class` gateway class that the Tigera Operator provisions by default. Possible `GatewayClass` customizations include the following: + - `gatewayDeployment.spec.template.metadata` allows arbitrary labels or annotations to be added to the pod that is created to implement each Gateway within that class. + - `gatewayDeployment.spec.template.spec.nodeSelector` allows control over where gateway implementation pods are scheduled. + - `gatewayDeployment.spec.template.spec.containers` allows control over the memory and CPU that the gateway implementation pods can use. + - `gatewayService.metadata.annotations` allows control over annotations to place on the service that is provisioned for each Gateway; these can be used, for example, to configure the cloud-specific type and properties of the associated external load balancer. + - `gatewayService.spec.*loadBalancer*` allows control over the corresponding `*loadBalancer*` fields in the service that is provisioned for each gateway; in some clouds these can also be used to configure the type and properties of the associated external load balancer. + +For full details, see [the `GatewayAPI` reference documentation](../../reference/installation/api.mdx#gatewayapi). + +:::note + +Merged gateways are not supported. +If a custom `EnvoyProxy` sets `mergeGateways: true`, the operator ignores that setting and uses `false` instead, so each `Gateway` gets its own proxy and load balancer. + +::: + +To make use of these customization fields, use `kubectl edit gatewayapi tigera-secure` to edit the YAML for the `GatewayAPI` resource, and add or modify the customization fields that you require. + +### Customization examples + +This section contains examples of common customizations. + +#### Handle high traffic with multiple gateway pods and adequate memory per-pod + +```bash +kubectl apply -f - < .yaml + ``` + Replace `` with a name for your custom configuration. + ```yaml title='Example of a default Envoy configuration' + apiVersion: v1 + data: + envoy-gateway.yaml: | + apiVersion: gateway.envoyproxy.io/v1alpha1 + extensionApis: + enableBackend: true + enableEnvoyPatchPolicy: true + gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + kind: EnvoyGateway + logging: + level: + default: info + provider: + kubernetes: + rateLimitDeployment: + container: + image: quay.io/calico/envoy-ratelimit:v3.31.1 + patch: + type: StrategicMerge + value: + spec: + template: + spec: + containers: + - imagePullPolicy: IfNotPresent + name: envoy-ratelimit + pod: {} + shutdownManager: + image: quay.io/calico/envoy-gateway:v3.31.1 + type: Kubernetes + kind: ConfigMap + metadata: + creationTimestamp: "2025-11-18T12:34:04Z" + labels: + app.kubernetes.io/instance: tigera-gateway-api + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: gateway-helm + app.kubernetes.io/version: v1.5.0 + helm.sh/chart: gateway-helm-v1.5.0 + name: envoy-gateway-config + namespace: calico-system + ownerReferences: + - apiVersion: operator.tigera.io/v1 + blockOwnerDeletion: true + controller: true + kind: GatewayAPI + name: default + uid: ace36d13-3afc-4c52-b894-dc2b322bc36e + resourceVersion: "1978" + uid: 8135e15d-cc91-4e02-ab97-2910cff56008 + ``` +1. Remove the following cluster-generated metadata fields: + * `creationTimestamp` + * `ownerReferences` + * `resourceVersion` + * `uid` +1. Change `metadata.name` from the default `envoy-gateway-config` to something else. +1. Add your customizations to the `data.envoy-gateway.yaml` section. + + ```yaml title='Example of a customized Envoy configuration file with Jaeger integration' + apiVersion: v1 + data: + envoy-gateway.yaml: | + apiVersion: gateway.envoyproxy.io/v1alpha1 + extensionApis: + enableBackend: true + enableEnvoyPatchPolicy: true + gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + kind: EnvoyGateway + logging: + level: + default: info + provider: + kubernetes: + rateLimitDeployment: + container: + image: quay.io/calico/envoy-ratelimit:v3.31.1 + patch: + type: StrategicMerge + value: + spec: + template: + spec: + containers: + - imagePullPolicy: IfNotPresent + name: envoy-ratelimit + pod: {} + shutdownManager: + image: quay.io/calico/envoy-gateway:v3.31.1 + type: Kubernetes + // highlight-start + tracing: + enabled: true + provider: + type: Jaeger + jaeger: + host: jaeger-collector.observability.svc.cluster.local + port: 14268 + // highlight-end + kind: ConfigMap + metadata: + labels: + app.kubernetes.io/instance: tigera-gateway-api + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/name: gateway-helm + app.kubernetes.io/version: v1.5.0 + helm.sh/chart: gateway-helm-v1.5.0 + // highlight-next-line + name: custom-envoy-config + namespace: calico-system + ``` + +1. Apply the customized Envoy configuration: + ```bash + kubectl apply -f custom-envoy-config.yaml + ``` +1. Patch the `GatewayAPI` resource to include the path of the custom Envoy configuration: + ```bash + kubectl patch gatewayapi tigera-secure --type='json' -p='[{"op": "replace", "path": "/spec/envoyGatewayConfigRef", "value": {"name": "custom-envoy-config", "namespace": "calico-system"}}]' + ``` + +## Additional resources + +* `GatewayAPI` spec in the [Tigera Operator API reference](../../reference/installation/api.mdx#gatewayapi) +* [Envoy Gateway documentation](https://gateway.envoyproxy.io/docs/) \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/networking/ingress-gateway/tutorial-ingress-gateway-canary.mdx b/calico-cloud_versioned_docs/version-24-1/networking/ingress-gateway/tutorial-ingress-gateway-canary.mdx new file mode 100644 index 0000000000..da4a4c9e2e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/ingress-gateway/tutorial-ingress-gateway-canary.mdx @@ -0,0 +1,416 @@ +--- +description: Step-by-step tutorial for running a canary rollout in Calico Cloud by splitting HTTPRoute weights across two backends behind an ingress gateway. +title: "Tutorial: Canary deployment" +--- + +import CodeBlock from '@theme/CodeBlock'; +export const calicoVersion='v3.31.1' +export const files=`https://raw.githubusercontent.com/projectcalico/calico/${calicoVersion}`; + + +# Tutorial: Launch a canary deployment with Calico Ingress Gateway + +This tutorial shows you how to create a canary deployment by using Calico Ingress Gateway. + +## Overview + +A **canary deployment** is a progressive release strategy where a new version of an application is gradually rolled out to a small subset of users before being fully deployed. +This approach allows teams to detect issues early, validate new features, and roll back quickly if needed, minimizing disruption to the majority of users. + +In $[prodname], canary deployments are implemented with a **traffic splitting** method that is defined in a routing resource associated with an ingress gateway. +By defining routing rules, you direct a small proportion of your incoming traffic to the canary version of your workload for testing. +The majority of the traffic continues to the stable version. + +As you become more confident in the canary version, you increase the proportion of traffic going to that version incrementally until it becomes the new stable version. + +## Use cases + +- **Feature Rollouts:** Safely release a new feature to a small subset of users to validate functionality and gather feedback before full deployment. +- **Performance Testing:** Evaluate how a new version handles real-world traffic and resource usage without affecting all users. +- **Bug Detection:** Catch critical bugs or unexpected behavior in production with minimal user impact. +- **A/B Testing:** Test different versions of a service to compare performance, user engagement, or conversion rates before committing to one. +- **Compliance and Risk Management:** Gradually roll out updates in highly regulated or mission-critical environments to reduce the chance of widespread failures. + +Canary deployments are ideal when you want controlled, low-risk releases, allowing teams to monitor metrics, validate assumptions, and ensure stability before a full-scale rollout. + +## Before you begin + +You'll need to install a few tools to complete this tutorial: + +* `kind`. + This is what you'll use to create a cluster on your workstation. + For installation instructions, see the [`kind` documentation](https://kind.sigs.k8s.io/docs/user/quick-start/). +* Docker Engine or Docker Desktop. + This is required to run containers for the `kind` utility. + For installation instructions, see the [Docker documentation](https://docs.docker.com/desktop/). +* `kubectl`. + This is the tool you'll use to interact with your cluster. + For installation instructions, see the [Kubernetes documentation](https://kubernetes.io/docs/tasks/tools/#kubectl) + +## Step 1: Set up your environment + +For this tutorial, you need access to a cluster with Calico Open Source 3.33 or later installed. + +We'll use `kind` to create this environment, but you can use any other supported Kubernetes distribution with a compatible version of $[prodname]. +If you've already got a suitable environment, start the tutorial at [Step 2: Create the ingress gateway](#step-2-create-an-ingress-gateway). + +1. Create a `kind` cluster with one control-plane node and two worker nodes. + + ```bash + cat > config.yaml < 5m23s v1.33.1 + calico-cluster-worker2 NotReady 5m22s v1.33.1 + ``` + + The nodes remain in a `NotReady` status until you configure networking in the next step. + +1. Install Calico Open Source by adding custom resource definitions, the Tigera Operator, and the custom resources. + + {`kubectl create -f ${files}/manifests/operator-crds.yaml + kubectl create -f ${files}/manifests/tigera-operator.yaml + kubectl create -f ${files}/manifests/custom-resources.yaml`} + +1. Monitor the deployment by running the following command: + + ```bash + watch kubectl get tigerastatus + ``` + + After a few minutes, all the Calico components display `True` in the `AVAILABLE` column. + + ```bash title="Expected output" + NAME AVAILABLE PROGRESSING DEGRADED SINCE + apiserver True False False 4m9s + calico True False False 3m29s + goldmane True False False 3m39s + ippools True False False 6m4s + whisker True False False 3m19s + ``` + +## Step 2. Create an ingress gateway + +Enable the Gateway API by creating a `GatewayAPI` resource. +When you create this resource, the Tigera Operator pulls Envoy Gateway images and sets things up. +Once that is set up, you can create supported Gateway API resources, such as the `Gateway` and routing resources. + +1. To enable Gateway API support, create a `GatewayAPI` resource with name `default`: + + ```bash + kubectl apply -f - <

App Version 1

+ --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: app-v1 + namespace: ingress-gateway-demo + spec: + replicas: 1 + selector: + matchLabels: + app: ingress-gateway-demo + version: v1 + template: + metadata: + labels: + app: ingress-gateway-demo + version: v1 + spec: + containers: + - name: busybox + image: busybox:latest + command: ["sh", "-c", "httpd -f -p 80 -h /var/www/html"] + ports: + - containerPort: 80 + volumeMounts: + - name: html + mountPath: /var/www/html + volumes: + - name: html + configMap: + name: app-v1-html + --- + apiVersion: v1 + kind: Service + metadata: + name: app-v1 + namespace: ingress-gateway-demo + spec: + selector: + app: ingress-gateway-demo + version: v1 + ports: + - port: 80 + targetPort: 80 + EOF + ``` + ```bash title='Expected output' + configmap/app-v1-html created + deployment.apps/app-v1 created + service/app-v1 created + ``` +1. Now deploy the same resources for the canary build, ingress-gateway-demo v2: + ```bash + cat << EOF | kubectl apply -f - + apiVersion: v1 + kind: ConfigMap + metadata: + name: app-v2-html + namespace: ingress-gateway-demo + data: + index.html: | +

App Version 2

+ --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: app-v2 + namespace: ingress-gateway-demo + spec: + replicas: 1 + selector: + matchLabels: + app: ingress-gateway-demo + version: v2 + template: + metadata: + labels: + app: ingress-gateway-demo + version: v2 + spec: + containers: + - name: busybox + image: busybox:latest + command: ["sh", "-c", "httpd -f -p 80 -h /var/www/html"] + ports: + - containerPort: 80 + volumeMounts: + - name: html + mountPath: /var/www/html + volumes: + - name: html + configMap: + name: app-v2-html + --- + apiVersion: v1 + kind: Service + metadata: + name: app-v2 + namespace: ingress-gateway-demo + spec: + selector: + app: ingress-gateway-demo + version: v2 + ports: + - port: 80 + targetPort: 80 + EOF + ``` + + ```bash title='Expected output' + configmap/app-v2-html created + deployment.apps/app-v2 created + service/app-v2 created + ``` + +## Step 4: Define an `HTTPRoute` resource to split traffic between app-v1 and app-v2 + +The `HTTPRoute` below routes 80% of requests to app-v1 and 20% to app-v2 + + ```bash + cat << EOF | kubectl apply -f - + apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + name: traffic-splitting + namespace: ingress-gateway-demo + spec: + parentRefs: + - name: canary-deployment-gateway + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: app-v1 + port: 80 + weight: 80 + - name: app-v2 + port: 80 + weight: 20 + EOF + ``` + +## Step 5. Test the canary deployment + +For this tutorial, we'll try accessing the app through the gateway by port-forwarding to your local workstation. +In a real-world scenario, you'd set up a load balancer to direct external traffic. + +1. Set up port forwarding to the gateway's service, which runs in the `ingress-gateway-demo` namespace: + + ```bash + kubectl port-forward -n ingress-gateway-demo svc/canary-deployment-gateway 30135:80 + ``` + Now you can access the app in your web browser at http://localhost:30135. + + +1. Now, in another terminal, you can test the canary build by accessing the app and checking how often each version is used: + + ```bash + for i in $(seq 200); do curl -s http://localhost:30135/ | grep "

"; done | \ + awk ' + /App Version 1/ {v1++} + /App Version 2/ {v2++} + END { + total=v1+v2 + printf("Version 1: %d (%.1f%%)\nVersion 2: %d (%.1f%%)\n", + v1, v1/total*100, v2, v2/total*100) + }' + ``` + + ```bash title='Sample output' + Version 1: 161 (80.5%) + Version 2: 39 (19.5%) + ``` + +## Step 6. Clean up your environment + +* To clean up your tutorial environment, run the following command: + + ```bash + kind delete cluster --name calico-cluster + ``` \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/networking/ipam/assign-ip-addresses-topology.mdx b/calico-cloud_versioned_docs/version-24-1/networking/ipam/assign-ip-addresses-topology.mdx new file mode 100644 index 0000000000..390e4ebafc --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/ipam/assign-ip-addresses-topology.mdx @@ -0,0 +1,181 @@ +--- +description: Bind Calico Cloud IP pools to zones, racks, or regions in a connected cluster with node selectors so pods get addresses that match the topology. +--- + +# Assign IP addresses based on topology + +## Big picture + +Assign blocks of IP addresses from an IP pool for different topological areas. + +## Value + +If you have workloads in different regions, zones, or rack, you may want them to get IP addresses from the same IP pool. This strategy is useful for reducing the number of routes that are required in the network, or to meet requirements imposed by an external firewall device or policy. $[prodname] makes it easy to do this using an IP pool resource with node labels and node selectors. + +## Concepts + +### IP address assignment + +Topology-based IP address assignment requires addresses to be per-host (node). +As such, Kubernetes annotations cannot be used because annotations are only per-namespace and per-pod. And although you can configure IP addresses for nodes in the CNI configuration, you are making changes within the host’s file system. The best option is to use node-selection IP address assignment using IP pools. + +### Node-selection IP address management + +Node selection-based IP address assignment is exactly what it sounds like: node labels are set, and Calico uses node selectors to decide which IP pools to use when assigning IP addresses to the node. + +### Best practice + +Nodes only assign workload addresses from IP pools which select them. To avoid having a workload not get an IP and fail to start, it is important to ensure that all nodes are selected by at least one IP pool. + +## How to + +### Create an IP pool, specific nodes + +In the following example, we create an IP pool that only allocates IP addresses for nodes with the label, **zone=west**. + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: zone-west-ippool +spec: + cidr: 192.168.0.0/24 + ipipMode: Always + natOutgoing: true + nodeSelector: zone == "west" +``` + +Then, we label a node with zone=west. For example: + +```bash +kubectl label nodes kube-node-0 zone=west +``` + +## Tutorial + +In this tutorial, we create a cluster with four nodes across two racks (two nodes/rack). + +``` + ------------------- + | router | + ------------------- + | | +--------------- --------------- +| rack-0 | | rack-1 | +--------------- --------------- +| kube-node-0 | | kube-node-2 | +- - - - - - - - - - - - - - - - +| kube-node-1 | | kube-node-3 | +- - - - - - - - - - - - - - - - +``` + +Using the pod IP range `192.168.0.0/16`, we target the following setup: reserve +the `192.168.0.0/24` and `192.168.1.0/24` pools for `rack-0`, `rack-1`. Let's +get started. + +By installing $[prodname] without setting the default IP pool to match, +running `calicoctl get ippool -o wide` shows that $[prodname] created its +default IP pool of `192.168.0.0/16`: + +``` +NAME CIDR NAT IPIPMODE DISABLED SELECTOR +default-ipv4-ippool 192.168.0.0/16 true Always false all() +``` + +1. Delete the default IP pool. + + Since the `default-ipv4-ippool` IP pool resource already exists and accounts + for the entire `/16` block, we will have to delete this first: + + ```bash + kubectl delete ippools default-ipv4-ippool + ``` + +2. Label the nodes. + + To assign IP pools to specific nodes, these nodes must be labelled + using [kubectl label](https://kubernetes.io/docs/tasks/configure-pod-container/assign-pods-nodes/#add-a-label-to-a-node). + + ```bash + kubectl label nodes kube-node-0 rack=0 + kubectl label nodes kube-node-1 rack=0 + kubectl label nodes kube-node-2 rack=1 + kubectl label nodes kube-node-3 rack=1 + ``` + +3. Create an IP pool for each rack. + + ```bash + kubectl create -f -< + nginx-5c7588df-s7qw6 1/1 Running 0 6m7s 192.168.0.129 kube-node-1 + nginx-5c7588df-w7r7g 1/1 Running 0 6m3s 192.168.1.65 kube-node-2 + nginx-5c7588df-62lnf 1/1 Running 0 6m3s 192.168.1.1 kube-node-3 + nginx-5c7588df-pnsvv 1/1 Running 0 6m3s 192.168.1.64 kube-node-2 + ``` + + The grouping of IP addresses assigned to the workloads differ based on what + node that they were scheduled to. Additionally, the assigned address for + each workload falls within the respective IP pool that selects the rack that + they run on. + +:::note + +$[prodname] IPAM will not reassign IP addresses to workloads +that are already running. To update running workloads with IP addresses from +a newly configured IP pool, they must be recreated. We recommend doing this +before going into production or during a maintenance window. + +::: + +## Additional resources + +[Calico IPAM](../../reference/component-resources/configuration.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/ipam/change-block-size.mdx b/calico-cloud_versioned_docs/version-24-1/networking/ipam/change-block-size.mdx new file mode 100644 index 0000000000..92fc1e5883 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/ipam/change-block-size.mdx @@ -0,0 +1,262 @@ +--- +description: Resize a Calico Cloud IPPool block — by creating a replacement pool and migrating workloads — to use IP space more efficiently across connected clusters. +--- + +# Change IP pool block size + +import DetermineIpam from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_determine-ipam.mdx'; + +## Big picture + +Change the IP pool block size to efficiently manage IP pool addresses. + +## Value + +Changing IP pool block size after installation requires ordered steps to minimize pod connectivity disruption. + +## Concepts + +### About IP pools + +By default, $[prodname] uses an IPAM block size of 64 addresses – /26 for IPv4, and /122 for IPv6. However, the block size can be changed depending on the IP pool address family. + +- IPv4: 20-32, inclusive +- IPv6: 116-128, inclusive + +You can have **only one default IP pool for per protocol** in your installation manifest. In this example, there is one IP pool for IPv4 (/26), and one IP pool for IPv6 (/122). + +```yaml +apiVersion: operator.tigera.io/v1 +kind: Installation +metadata: + name: default +spec: + # Configures Calico networking. + calicoNetwork: + ipPools: + - blockSize: 26 + cidr: 10.48.0.0/21 + encapsulation: IPIP + natOutgoing: Enabled + nodeSelector: all() + - blockSize: 122 + cidr: 2001::00/64 + encapsulation: None + natOutgoing: Enabled + nodeSelector: all() +``` + +However, the following is invalid because it has two IP pools for IPv4. + +```yaml +apiVersion: operator.tigera.io/v1 +kind: Installation +metadata: + name: default +spec: + # Configures Calico networking. + calicoNetwork: + ipPools: + - blockSize: 26 + cidr: 10.48.0.0/21 + encapsulation: IPIP + natOutgoing: Enabled + nodeSelector: all() + - blockSize: 31 + cidr: 10.48.8.0/21 + encapsulation: IPIP + natOutgoing: Enabled + nodeSelector: all() +``` + +### Expand or shrink IP pool block sizes + +By default, the $[prodname] IPAM block size for an IP pool is /26. To expand from the default size /26, lower the `blockSize` (for example, /24). To shrink the `blockSize` from the default /26, raise the number (for example, /28). + +### Best practice: change IP pool block size before installation + +Because the `blockSize` field cannot be edited directly after $[prodname] installation, it is best to change the IP pool block size before installation to minimize disruptions to pod connectivity. + +## Before you begin... + +**Required** + +Verify that you are using $[prodname] IPAM. + + + +## How to + +:::note + +Follow the steps to minimize pod connectivity disruption. Pods may lose connectivity when they are redeployed, and may lose external connectivity while in the temporary pool. Also, when pods are deleted, applications may be temporarily unavailable (depending on the type of application). Plan your changes accordingly. + +::: + +The high-level steps to follow are: + +1. [Create a temporary IP pool](#create-a-temporary-ip-pool) + + **Note**: The temporary IP pool must not overlap with the existing one. + +1. [Disable the existing IP pool](#disable-the-existing-ip-pool) + + **Note**: When you disable an IP pool, only new IP address allocations are prevented; networking of existing pods are not affected. + +1. [Delete pods from the existing IP pool](#delete-pods-from-the-existing-ip-pool) + This includes any new pods that may have been created with the existing IP pool prior to disabling the pool. Verify that new pods get an address from the temporary IP pool. +1. [Delete the existing IP pool](#delete-the-existing-ip-pool) +1. [Create a new IP pool with the desired block size](#create-a-new-ip-pool-with-the-desired-block-size) +1. [Disable the temporary IP pool](#disable-the-temporary-ip-pool) +1. [Delete pods from the temporary IP pool](#delete-pods-from-the-temporary-ip-pool) +1. [Delete the temporary IP pool](#delete-the-temporary-ip-pool) + +## Tutorial + +In the following steps, our Kubernetes cluster has a default CIDR block size of /26. We want to shrink the block size to /28 to use the pool more efficiently. + +### Create a temporary IP pool + +We add a new IPPool with the CIDR range, 10.0.0.0/16. + +Create a temporary-pool.yaml. + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: temporary-pool +spec: + cidr: 10.0.0.0/16 + ipipMode: Always + natOutgoing: true +``` + +Apply the changes. + +```bash +kubectl apply -f temporary-pool.yaml +``` + +Let’s verify the temporary IP pool. + +```bash +calicoctl get ippool -o wide +``` + +``` +NAME CIDR NAT IPIPMODE DISABLED +default-ipv4-ippool 192.168.0.0/16 true Always false +temporary-pool 10.0.0.0/16 true Always false +``` + +### Disable the existing IP pool + +Disable allocations in the default pool. + +```bash +kubectl patch ippool default-ipv4-ippool -p '{"spec": {"disabled": "true"}}' +``` + +Verify the changes. + +```bash +calicoctl get ippool -o wide +``` + +``` +NAME CIDR NAT IPIPMODE DISABLED +default-ipv4-ippool 192.168.0.0/16 true Always true +temporary-pool 10.0.0.0/16 true Always false +``` + +### Delete pods from the existing IP pool + +In our example, **coredns** is our only pod; for multiple pods you would trigger a deletion for all pods in the cluster. + +```bash +kubectl delete pod -n kube-system coredns-6f4fd4bdf-8q7zp +``` + +Restart all pods with just one command. + +:::caution + +The following command is disruptive and may take several minutes depending on the number of pods deployed. + +::: + +```bash +kubectl delete pod -A --all +``` + +### Delete the existing IP pool + +Now that you’ve verified that pods are getting IPs from the new range, you can safely delete the existing pool. + +```bash +kubectl delete ippool default-ipv4-ippool +``` + +### Create a new IP pool with the desired block size + +In this step, we update the IPPool with the new block size of (/28). + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: default-ipv4-ippool +spec: + blockSize: 28 + cidr: 192.0.0.0/16 + ipipMode: Always + natOutgoing: true +``` + +Apply the changes. + +```bash +kubectl apply -f pool.yaml +``` + +### Disable the temporary IP pool + +```bash +kubectl patch ippool temporary-pool -p '{"spec": {"disabled": "true"}}' +``` + +### Delete pods from the temporary IP pool + +In our example, **coredns** is our only pod; for multiple pods you would trigger a deletion for all pods in the cluster. + +```bash +kubectl delete pod -n kube-system coredns-6f4fd4bdf-8q7zp +``` + +Restart all pods with just one command. + +:::caution + +The following command is disruptive and may take several minutes depending on the number of pods deployed. + +::: + +```bash +kubectl delete pod -A --all +``` + +Validate your pods and block size are correct by running the following commands: + +```bash +kubectl get pods --all-namespaces -o wide +calicoctl ipam show --show-blocks +``` + +### Delete the temporary IP pool + +Clean up the IP pools by deleting the temporary IP pool. + +```bash +kubectl delete pool temporary-pool +``` diff --git a/calico-cloud_versioned_docs/version-24-1/networking/ipam/get-started-ip-addresses.mdx b/calico-cloud_versioned_docs/version-24-1/networking/ipam/get-started-ip-addresses.mdx new file mode 100644 index 0000000000..4d8f23d6cb --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/ipam/get-started-ip-addresses.mdx @@ -0,0 +1,97 @@ +--- +description: Pick between Calico Cloud IPAM and host-local IPAM, then configure pool selection, NAT outgoing, and per-namespace IP assignment in a connected cluster. +--- + +# Get started with IP address management + +## Big picture + +Understand how IP address management (IPAM) functions in a Kubernetes cluster using $[prodname]. + +## Value + +Different IPAM techniques provide different feature sets. $[prodname]’s IPAM provides additional IP allocation efficiency and flexibility compared to other address management approaches. + +## Concepts + +### IPAM in Kubernetes + +Kubernetes uses IPAM plugins to allocate and manage IP addresses assigned to pods. Different IPAM plugins provide different feature sets. $[prodname] provides its own IPAM plugin called **calico-ipam** which is designed to work well with $[prodname] and includes a number of features. + +### $[prodname] IPAM + +The **calico-ipam** plugin uses $[prodname]’s IP pool resource to control how IP addresses are allocated to pods within the cluster. This is the default plugin used by most $[prodname] installations. + +By default, $[prodname] uses a single IP pool for the entire Kubernetes pod CIDR, but you can divide the pod CIDR into several pools. You can assign separate IP pools to particular selections of **nodes**, or to teams, users, or applications within a cluster using **namespaces**. + +:::tip + +Because Calico IPAM does not use the Kubernetes node CIDR allocations (`Node.spec.podCIDR`), you can set `--allocate-node-cidrs=false` on the kube-controller-manager. +This prevents Kubernetes from allocating unused node CIDRs and avoids `CIDRNotAvailable` events on nodes. + +If you bootstrapped your cluster with kubeadm and passed `--pod-network-cidr`, kubeadm enables node CIDR allocation automatically. +See the [Kubernetes documentation](https://kubernetes.io/docs/tasks/administer-cluster/kubeadm/kubeadm-reconfigure/) for instructions on reconfiguring the controller manager after cluster creation. + +::: + +You can control which pools $[prodname] uses for each pod using + +- node selectors +- an annotation on the pod’s namespace, or +- an annotation on the pod + +$[prodname] also supports the **host-local** IPAM plugin. However, when using the host-local IPAM plugin some $[prodname] features are not available. + +### $[prodname] IPAM blocks + +In $[prodname] IPAM, IP pools are subdivided into blocks -- smaller chunks that are associated with a particular node in the cluster. Each node in the cluster can have one or more blocks associated with it. $[prodname] will automatically create and destroy blocks as needed as the number of nodes and pods in the cluster grows or shrinks. + +Blocks allow $[prodname] to efficiently aggregate addresses assigned to pods on the same node, reducing the size of the routing table. By default $[prodname] will try to allocate IP addresses from within an associated block, creating a new block if necessary. $[prodname] can also assign addresses to pods on a node that are not within a block associated with that node. This allows for IP allocations independent of the node on which a pod is launched. + +By default, $[prodname] creates blocks with room for 64 addresses (a /26), but you can control block sizes for each IP pool. + +### Host-local IPAM + +The host-local plugin is a simple IP address management plugin. It uses predetermined CIDRs statically allocated to each node to choose addresses for pods. Once set, the CIDR for a node cannot be modified. Pods can be assigned addresses only from within the CIDR allocated to the node. + +$[prodname] can use the host-local IPAM plugin, using the **Node.Spec.PodCIDR** field in the Kubernetes API to determine the CIDR to use for each node. However, per-node, per-pod, and per-namespace IP allocation features are not available using the host-local plugin. + +The host-local IPAM plugin is primarily used by other methods of routing pod traffic from one host to another. For example, it is used when installing $[prodname] for policy enforcement with flannel networking, as well as when using $[prodname] in Google Kubernetes Engine (GKE). + +## Before you begin + +**Limitations** + +- Works for platforms that use the Calico CNI + +## How to + +### Install $[prodname] with calico-ipam + +Follow one of the getting started guides to install $[prodname]. +{/* TODO-XREFS-CC */} + +### Install $[prodname] with host-local IPAM + +Follow one of the getting started guides to install $[prodname]. +{/* TODO-XREFS-CC */} + +Or, see the [reference documentation on host-local IPAM](../../reference/component-resources/configuration.mdx#using-host-local-ipam). + +## Tutorial + +For a blog/tutorial on IP pools, see [Calico Enterprise IPAM: Explained and Enhanced](https://www.tigera.io/blog/calico-ipam-explained-and-enhanced/). + +## Additional resources + +- [IP Pool](../../reference/resources/ippool.mdx) + +There are several other ways to leverage $[prodname] IPAM including: + +- [Assign addresses based on topology](assign-ip-addresses-topology.mdx) +- [Use a specific address for a pod](use-specific-ip.mdx) +- [Migrate from one IP pool to another](migrate-pools.mdx) +- [Restrict a pod to use an IP address in a specific range](legacy-firewalls.mdx) +- [Change IP address block size](../../reference/resources/ippool.mdx) + +{/*TODO-XREFS-CC - [View IP address utilization](/reference/clis/calicoctl/ipam/show/)*/} diff --git a/calico-cloud_versioned_docs/version-24-1/networking/ipam/index.mdx b/calico-cloud_versioned_docs/version-24-1/networking/ipam/index.mdx new file mode 100644 index 0000000000..69241e3d04 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/ipam/index.mdx @@ -0,0 +1,11 @@ +--- +description: IP address management in Calico Cloud — IPPools, block sizes, IPv6 dual stack, service load balancer IPAM, topology-aware allocation, and pool migration. +hide_table_of_contents: true +--- + +# IP address management + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/networking/ipam/initial-ippool.mdx b/calico-cloud_versioned_docs/version-24-1/networking/ipam/initial-ippool.mdx new file mode 100644 index 0000000000..2ce80d32ca --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/ipam/initial-ippool.mdx @@ -0,0 +1,71 @@ +--- +description: Set the default Calico Cloud IP pool CIDR, encapsulation, and NAT outgoing values during Tigera Operator installation through the Installation resource. +--- + +# Configure default IP pools + +## Big picture + +Configure the default IP pool values to use during Tigera Operator installation. + +## Value + +During Tigera Operator installation, you must configure the CIDR range to use for pods that reflects your environment. + +## Concepts + +### Kubernetes pod CIDR + +The **Kubernetes pod CIDR** is the expected IP address range for pod IPs. It is defined for the entire cluster, and is used by various Kubernetes components to determine if an IP belongs to a pod. For example, kube-proxy treats traffic differently if an IP is from a pod than if it is not. All pod IPs must be in the CIDR range for Kubernetes to function correctly. + +### Tigera Operator and IP pools + +[Calico IP pools](../../reference/resources/ippool.mdx) are ranges of IP addresses that Calico uses to assign to pods; the ranges must within the Kubernetes pod CIDR. + +The Tigera Operator reads the [Installation](../../reference/installation/api.mdx#installation) +resource and configures the default Calico IP pool. Note the following: + +- Default fields for any that are omitted: + - CIDR: 192.168.0.0/16 + - Encapsulation: IPIP + - NodeSelector: all() + - NATOutgoing: Enabled +- IP pools are only used when Calico is used for pod networking, IP pools are not utilized when using other pod networking solutions. +- To make changes to the IP pools after Tigera Operator install, you may use calicoctl or kubectl. If you make the changes to the IP Pool in the Installation resource (Operator IPPool) after installation, the changes are not applied. + +## Before you begin... + +- Verify that your IP pool is within the Kubernetes pod CIDR +- If you are using encapsulation (IP in IP or VXLAN), ensure that the traffic is allowed on your network +- You are making these changes for a cluster that has not yet had $[prodname] deployed. + +## How to + +1. Download the custom-resource.yaml file. +1. Edit the [Installation resource](../../reference/installation/api.mdx#installation). + **Required values**: `cidr:` + **Empty values**: Defaulted + + ```bash + apiVersion: operator.tigera.io/v1 + kind: Installation + metadata: + name: default + spec: + calicoNetwork: + ipPools: + - cidr: "192.168.0.0/16" + encapsulation: "IPIP" + nodeSelector: "label == 'value'" + natOutgoing: "Enabled" + ``` + +1. Apply the manifest and continue with your installation as normal. + +## Additional resources + +- [IP pool resource](../../reference/resources/ippool.mdx) + +{/*TODO-XREFS-CC +- Use [calicoctl](/reference/clis/calicoctl/index) or `kubectl` to edit the IPPool resource. +*/} diff --git a/calico-cloud_versioned_docs/version-24-1/networking/ipam/ip-autodetection.mdx b/calico-cloud_versioned_docs/version-24-1/networking/ipam/ip-autodetection.mdx new file mode 100644 index 0000000000..6f67759e2f --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/ipam/ip-autodetection.mdx @@ -0,0 +1,240 @@ +--- +description: Pick how Calico Cloud detects each node's primary IP — first-found, Kubernetes internal, interface regex, CIDR, or skip-interface — for reliable routing. +--- + +# Configure IP autodetection + +## Big picture + +Configure IP autodetection for $[prodname] nodes to ensure the correct IP address is used for routing. + +## Value + +When you install $[prodname] on a node, an IP address and subnet is automatically detected. $[prodname] provides several ways to configure IP/subnet autodetection, and supports configuring specific IPs for: + +- Hosts with multiple external interfaces +- Host interfaces with multiple IP addresses +- [Changes to cross subnet packet encapsulation](../configuring/vxlan-ipip.mdx) +- Changes to host IP address + +## Concepts + +### Autodetecting node IP address and subnet + +For internode routing, each $[prodname] node must be configured with an IPv4 address and/or an IPv6 address. When you install $[prodname] on a node, a node resource is automatically created using routing information that is detected from the host. For some deployments, you may want to update autodetection to ensure nodes get the correct IP address. + +**Sample default node resource after installation** + +```yaml +apiVersion: projectcalico.org/v3 +kind: Node +metadata: + name: node-hostname +spec: + bgp: + asNumber: 64512 + ipv4Address: 10.244.0.1/24 + ipv6Address: 2000:db8:85a3::8a2e:370:7335/120 + ipv4IPIPTunnelAddr: 192.168.0.1 +``` + +### Autodetection methods + +By default, $[prodname] uses the **firstFound** method; the first valid IP address on the first interface (excluding local interfaces such as the docker bridge). However, you can change the default method to any of the following: + +- Address used by the node to reach a particular IP or domain (**canReach**) +- Address assigned to Kubernetes node (**kubernetes: InternalIP**) +- Regex to include matching interfaces (**interface**) +- Regex to exclude matching interfaces (**skipInterface**) +- A list of IP ranges in CIDR format to determine valid IP addresses on the node to choose from (**cidrs**) + +For help on autodetection methods, see +[NodeAddressAutodetection](../../reference/installation/api.mdx#nodeaddressautodetection) in the Tigera Operator API reference +and for more details see the [node configuration](../../reference/component-resources/node/configuration.mdx#ip-autodetection-methods) reference. + +### Manually configure IP address and subnet + +To manually configure an IP address and subnet, disable autodetection and update the node resources with the IP address. + +## How to + +- [Change the autodetection method](#change-the-autodetection-method) +- [Manually configure IP address and subnet for a node](#manually-configure-ip-address-and-subnet-for-a-node) + +### Change the autodetection method + +As noted previously, the default autodetection method is **first valid interface found** (firstFound). To use a different autodetection method, +configure the NodeAddressAutodetection field(s) in the Installation resource. You can update the Installation resource before applying it +during installation or edit it later with `kubectl edit installation default`. + +:::note + +To configure the default autodetection method for IPv6 for any of the below methods, use the field `nodeAddressAutodetectionV6`. + +::: + +- **Kubernetes Node IP** + + $[prodname] will select the first internal IP address listed in the Kubernetes node's `Status.Addresses` field. + + ```yaml + kind: Installation + apiVersion: operator.tigera.io/v1 + metadata: + name: default + spec: + calicoNetwork: + nodeAddressAutodetectionV4: + kubernetes: NodeInternalIP + ``` + +- **Source address used to reach an IP or domain name** + + $[prodname] will choose the IP address that is used to reach the given "can reach" IP address or domain. For example: + + ```yaml + kind: Installation + apiVersion: operator.tigera.io/v1 + metadata: + name: default + spec: + calicoNetwork: + nodeAddressAutodetectionV4: + canReach: 8.8.8.8 + ``` + +- **Including matching interfaces** + + $[prodname] will choose an address on each node from an interface that matches the given [regex](https://pkg.go.dev/regexp). + For example: + + ```yaml + kind: Installation + apiVersion: operator.tigera.io/v1 + metadata: + name: default + spec: + calicoNetwork: + nodeAddressAutodetectionV4: + interface: eth.* + ``` + +- **Excluding matching interfaces** + + $[prodname] will choose an address on each node from an interface that does not match the given [regex](https://pkg.go.dev/regexp). + For example: + + ```yaml + kind: Installation + apiVersion: operator.tigera.io/v1 + metadata: + name: default + spec: + calicoNetwork: + nodeAddressAutodetectionV4: + skipInterface: eth.* + ``` + +- **Including CIDRs** + + $[prodname] will select any IP address from the node that falls within the given CIDRs. For example: + + ```yaml + kind: Installation + apiVersion: operator.tigera.io/v1 + metadata: + name: default + spec: + calicoNetwork: + nodeAddressAutodetectionV4: + cidrs: + - '192.168.200.0/24' + ``` + +- **IPv4** + + ```yaml noValidation + apiVersion: operator.tigera.io/v1 + kind: Installation + metadata: + name: default + spec: + variant: CalicoEnterprise + ... + calicoNetwork: + nodeAddressAutodetectionV4: + : + ``` + +- **IPv6** + + ```yaml noValidation + apiVersion: operator.tigera.io/v1 + kind: Installation + metadata: + name: default + spec: + variant: CalicoEnterprise + ... + calicoNetwork: + nodeAddressAutodetectionV6: + : + ``` + +:::note + +You can use both `nodeAddressAutodetectionV4` and `nodeAddressAutodetectionV6` to specify IPv4 and IPv6 methods. + +::: + +### Manually configure IP address and subnet for a node + +In the following scenarios, you may want to configure a specific IP and subnet: + +- Hosts with multiple external interfaces +- Host interfaces with multiple IP addresses +- Changes to cross subnet packet encapsulation +- Changes to host IP address + +You can configure specific IP address and subnet for a node by disabling IP autodetection and then updating the [Node resource](../../reference/resources/node.mdx). + +#### Disable autodetection + +To disable autodetection method, update the proper `NodeAddressAutodetection` field in the Installation resource: + +```yaml noValidation +apiVersion: operator.tigera.io/v1 +kind: Installation +metadata: + name: default +spec: + variant: CalicoEnterprise + ... + calicoNetwork: + nodeAddressAutodetectionV4: {} + # nodeAddressAutodetectionV6: {} +``` + +#### Configure IP and subnet using node resource + +You can also configure the IP address and subnet on a Node resource. + +:::note + +When configuring the IP address on a Node resource, you may want to disable IP address options or environment variables on the node. IP options on the container take precedence, and will overwrite the values you configure on the node resource. + +::: + +Use `calicoctl patch` to update the current node configuration. For example: + +``` +calicoctl patch node kind-control-plane \ + --patch='{"spec":{"bgp": {"ipv4Address": "10.0.2.10/24", "ipv6Address": "fd80:24e2:f998:72d6::/120"}}}' +``` + +## Additional resources + +- For details on autodetection methods, see the [node configuration](../../reference/component-resources/node/configuration.mdx) reference. +- For calicoctl environment variables, see [Configuring $[nodecontainer]](../../reference/component-resources/node/configuration.mdx) +- [Node resource](../../reference/resources/node.mdx) +{/*//TODO-CC-XREFS- [Reference documentation for calicoctl patch](/reference/clis/calicoctl/patch/)*/} diff --git a/calico-cloud_versioned_docs/version-24-1/networking/ipam/ipv6.mdx b/calico-cloud_versioned_docs/version-24-1/networking/ipam/ipv6.mdx new file mode 100644 index 0000000000..f5c8749008 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/ipam/ipv6.mdx @@ -0,0 +1,94 @@ +--- +description: Configure dual-stack pod networking in a Calico Cloud connected cluster by adding IPv6 IP pools, IPv6 autodetection, and matching CNI plugin settings. +--- + +# Configure dual stack + +## Big picture + +Configure $[prodname] IP address allocation to use dual stack for workload communications. + +## Value + +In addition to IPv4, IPv6 is increasingly desirable for Workload communication. $[prodname] supports: + +- **IPv4 only** (default) + + Each workload gets an IPv4 address, and can communicate over IPv4. + +- **Dual stack** + + Each workload gets an IPv4 and an IPv6 address, and can communicate over IPv4 or IPv6. + +## Before you begin + +**Unsupported** + +- IPv6 only +- AKS +- AWS/kOps +- EKS +- GKE +- RKE +- RKE2 + +**$[prodname] requirements** + +- $[prodname] IPAM +- OpenShift + - Requires 4.8 for IPv6/IPv4 dual-stack and IPv6 single stack support + - Requires 3.11 and later using $[prodname] 3.4 and later for IPv6 support + +**Kubernetes version requirements** + +- For dual stack, 1.20 and later +- For one IP stack at a time (IPv4 or IPv6), any Kubernetes version + +**Kubernetes IPv6 host requirements** + +- An IPv6 address that is reachable from the other hosts +- The sysctl setting, `net.ipv6.conf.all.forwarding`, is set to `1`. + This ensures both Kubernetes service traffic and $[prodname] traffic is forwarded appropriately. +- A default IPv6 route + +**Kubernetes IPv4 host requirements** + +- An IPv4 address that is reachable from the other hosts +- The sysctl setting, `net.ipv4.conf.all.forwarding`, is set to `1`. + This ensures both Kubernetes service traffic and $[prodname] traffic is forwarded appropriately. +- A default IPv4 route + +## How to + +:::note + +The following task is only for new clusters. + +::: + +### Enable dual stack + +1. Set up a new cluster following the Kubernetes [prerequisites](https://kubernetes.io/docs/concepts/services-networking/dual-stack/#prerequisites) and [enablement steps](https://kubernetes.io/docs/concepts/services-networking/dual-stack/#enable-ipv4-ipv6-dual-stack). + +To configure dual-stack cluster using the operator, edit your default Installation at install time to include both an IPv4 and IPv6 pool. For example: + +```yaml +apiVersion: operator.tigera.io/v1 +kind: Installation +metadata: + name: default +spec: + # Configures Calico networking. + calicoNetwork: + ipPools: + - blockSize: 26 + cidr: 10.48.0.0/21 + encapsulation: IPIP + natOutgoing: Enabled + nodeSelector: all() + - blockSize: 122 + cidr: 2001::00/64 + encapsulation: None + natOutgoing: Enabled + nodeSelector: all() +``` diff --git a/calico-cloud_versioned_docs/version-24-1/networking/ipam/legacy-firewalls.mdx b/calico-cloud_versioned_docs/version-24-1/networking/ipam/legacy-firewalls.mdx new file mode 100644 index 0000000000..409b77b947 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/ipam/legacy-firewalls.mdx @@ -0,0 +1,66 @@ +--- +description: Restrict pods to a defined IP range in a Calico Cloud connected cluster so legacy firewalls and appliances can recognise cluster workloads by source IP. +--- + +# Restrict a pod to use an IP address in a specific range + +import DetermineIpam from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_determine-ipam.mdx'; + +## Big picture + +Restrict the IP address chosen for a pod to a specific range of IP addresses. + +## Value + +When Kubernetes pods interact with external systems that make decisions based on IP ranges (for example legacy firewalls), it can be useful to define several IP ranges and explicitly assign pods to those ranges. Using $[prodname] IP Address Management (IPAM), you can restrict a pod to use an address from within a specific range. + +## Concepts + +### Kubernetes pod CIDR + +The **Kubernetes pod CIDR** is the range of IPs Kubernetes expects pod IPs to be assigned from. It is defined for the entire cluster and is used by various Kubernetes components to determine whether an IP belongs to a pod. For example, kube-proxy treats traffic differently if that traffic is from a pod than if it is not. All pod IPs must be in the CIDR range for Kubernetes to function correctly. + +### IP Pool + +**IP pools** are ranges of IP addresses from which $[prodname] assigns pod IPs. By default, $[prodname] creates an IP pool for the entire Kubernetes pod CIDR, but you can change this to break the pod CIDR up into several pools. You can control which pool $[prodname] uses for each pod using node selectors, or annotations on the pod or the pod’s namespace. + +## Before you begin... + +The features in this How to guide require: + +- $[prodname] IPAM + + + +Additionally, cluster administrators must have [configured IP pools](../../reference/resources/ippool.mdx) to define the valid IP ranges to use for allocating pod IP addresses. + +## How to + +### Restrict a pod to use an IP address range + +Annotate the pod with key `cni.projectcalico.org/ipv4pools` and/or `cni.projectcalico.org/ipv6pools` and value set to a list of IP pool names, enclosed in brackets. For example: + +``` +cni.projectcalico.org/ipv4pools: '["pool-1", "pool-2"]' +``` + +Note the use of the escaped \" for the inner double quotes around the pool names. + +### Restrict all pods within a namespace to use an IP address range + +Annotate the namespace with key `cni.projectcalico.org/ipv4pools` and/or `cni.projectcalico.org/ipv6pools` and value set to a list of IP pool names, enclosed in brackets. For example: + +``` +cni.projectcalico.org/ipv4pools: '["pool-1", "pool-2"]' + +``` + +Note the use of the escaped `\"` for the inner double quotes around the pool names. + +If both the pod and the pod’s namespace have the annotation, the pod annotation takes precedence. + +The annotation must be present at the time the pod is created. Adding it to an existing pod has no effect. + +## Additional resources + +For help configuring $[prodname] IPAM, see [Configuring the $[prodname] CNI Plugins](../../reference/component-resources/configuration.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/networking/ipam/migrate-pools.mdx b/calico-cloud_versioned_docs/version-24-1/networking/ipam/migrate-pools.mdx new file mode 100644 index 0000000000..9b11e12bff --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/ipam/migrate-pools.mdx @@ -0,0 +1,228 @@ +--- +description: Move workloads from one Calico Cloud IPPool to another on a running connected cluster without disrupting existing pod connectivity. +--- + +# Migrate from one IP pool to another + +import DetermineIpam from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_determine-ipam.mdx'; + +## Big picture + +Migrate pods from one IP pool to another on a running cluster without network disruption. + +## Value + +Pods are assigned IP addresses from IP pools that you configure in $[prodname]. As the number of pods increase, you may need to increase the number of addresses available for pods to use. Or, you may need to move pods from a CIDR that was used by mistake. $[prodname] lets you migrate from one IP pool to another one on a running cluster without network disruption. + +## Concepts + +### IP pools and cluster CIDRs + +$[prodname] supports using multiple disjoint IP pool CIDRs within the cluster. However, Kubernetes expects that all pods have addresses within the same cluster CIDR. This means that although it is technically feasible to create an IP pool outside of the cluster CIDR, we do not recommend it. Pods allocated addresses outside of the Kubernetes cluster CIDR will lose network connectivity. + +## Before you begin... + +**Verify that you are using $[prodname] IPAM**. + + + +**Verify orchestrator support for changing the pod network CIDR**. + +Although Kubernetes supports changing the pod network CIDR, not all orchestrators do. For example, OpenShift does not support this feature as described in + +## How to + +### Migrate from one IP pool to another + +Follow these steps to migrate pods from one IP pool to another pool. + +:::note + +If you follow these steps, existing pod connectivity will not be affected. (If you delete the old IP pool before you create and verify the new pool, existing pods will be affected.) When pods are deleted, applications may be temporarily unavailable (depending on the type of application); plan accordingly. + +::: + +1. Add a new IP pool. + + :::note + + It is highly recommended that your Calico IP pools are within the Kubernetes cluster CIDR. If pods IPs are allocated from outside of the Kubernetes cluster CIDR, some traffic flows may have NAT applied unnecessarily causing unexpected behavior. + + If you have configured kube-proxy with a cluster CIDR, you must ensure that it contains the new IP pool CIDR in order to prevent the unwanted NAT. + + ::: + +1. Disable the old IP pool. + + :::note + + Disabling an IP pool only prevents new IP address allocations; it does not affect the networking of existing pods. + + ::: + +1. Delete pods from the old IP pool. This includes any new pods that may have been created with the old IP pool prior to disabling the pool. + +1. Verify that new pods get an address from the new IP pool. + +1. Delete the old IP pool. + +## Tutorial + +In the following example, we created a Kubernetes cluster using **kubeadm**. But the IP pool CIDR we configured (192.168.0.0/16) doesn't match the +Kubernetes cluster CIDR. Let's change the CIDR to **10.0.0.0/16**, which for the purposes of this example falls within the cluster CIDR. + +Let’s run `calicoctl get ippool -o wide` to see the IP pool, **default-ipv4-ippool**. + +``` +NAME CIDR NAT IPIPMODE VXLANMODE DISABLED +default-ipv4-ippool 192.168.0.0/16 true Always Never false +``` + +When we run `calicoctl get wep --all-namespaces`, we see that a pod is created using the default range (192.168.52.130/32). + +``` +NAMESPACE WORKLOAD NODE NETWORKS INTERFACE +kube-system coredns-6f4fd4bdf-8q7zp vagrant 192.168.52.130/32 cali800a63073ed +``` + +Let’s get started changing this pod to the new IP pool (10.0.0.0/16). + +### Step 1: Add a new IP pool + +We add a new **IPPool** with the CIDR range, **10.0.0.0/16**. + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: new-pool +spec: + cidr: 10.0.0.0/16 + ipipMode: Always + natOutgoing: true +``` + +Let’s verify the new IP pool. + +```bash +calicoctl get ippool -o wide + +``` + +``` +NAME CIDR NAT IPIPMODE DISABLED +default-ipv4-ippool 192.168.0.0/16 true Always false +new-pool 10.0.0.0/16 true Always false +``` + +### Step 2: Disable the old IP pool + +List the existing IP pool definition. + +```bash +calicoctl get ippool -o yaml > pools.yaml + +``` + +```yaml +apiVersion: projectcalico.org/v3 +items: + - apiVersion: projectcalico.org/v3 + kind: IPPool + metadata: + name: default-ipv4-ippool + spec: + cidr: 192.0.0.0/16 + ipipMode: Always + natOutgoing: true + - apiVersion: projectcalico.org/v3 + kind: IPPool + metadata: + name: new-pool + spec: + cidr: 10.0.0.0/16 + ipipMode: Always + natOutgoing: true +``` + +Edit pools.yaml. + +Disable this IP pool by setting: `disabled: true` + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: default-ipv4-ippool +spec: + cidr: 192.0.0.0/16 + ipipMode: Always + natOutgoing: true + disabled: true +``` + +Apply the changes. + +Remember, disabling a pool only affects new IP allocations; networking for existing pods is not affected. + +```bash +kubectl apply -f pools.yaml +``` + +Verify the changes. + +```bash +calicoctl get ippool -o wide +``` + +``` +NAME CIDR NAT IPIPMODE DISABLED +default-ipv4-ippool 192.168.0.0/16 true Always true +new-pool 10.0.0.0/16 true Always false +``` + +### Step 3: Delete pods from the old IP pool + +Next, we delete all of the existing pods from the old IP pool. (In our example, **coredns** is our only pod; for multiple pods you would trigger a deletion for all pods in the cluster.) + +```bash +kubectl delete pod -n kube-system coredns-6f4fd4bdf-8q7zp +``` + +### Step 4: Verify that new pods get an address from the new IP pool + +1. Create a test namespace and nginx pod. + + ```bash + kubectl create ns ippool-test + ``` + +1. Create an nginx pod. + + ```bash + kubectl -n ippool-test create deployment nginx --image nginx + ``` + +1. Verify that the new pod gets an IP address from the new range. + + ```bash + kubectl -n ippool-test get pods -l app=nginx -o wide + ``` + +1. Clean up the ippool-test namespace. + + ```bash + kubectl delete ns ippool-test + ``` + +### Step 5: Delete the old IP pool + +Now that you've verified that pods are getting IPs from the new range, you can safely delete the old pool. + +```bash +kubectl delete ippool default-ipv4-ippool +``` + +## Additional resources + +- [IP pools reference](../../reference/resources/ippool.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/ipam/service-loadbalancer.mdx b/calico-cloud_versioned_docs/version-24-1/networking/ipam/service-loadbalancer.mdx new file mode 100644 index 0000000000..7d8cd82caa --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/ipam/service-loadbalancer.mdx @@ -0,0 +1,149 @@ +--- +description: Use the Calico Cloud LoadBalancer controller to allocate IPs to Kubernetes Service type LoadBalancer from configured IPPool resources in a connected cluster. +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# LoadBalancer IP address management + +## Understanding $[prodname] LoadBalancer IP address management + +You might want to utilize Service of type LoadBalancer in your cluster to provide stable long-lasting IP address to access your deployed application. +$[prodname] can help you with providing and managing IP addresses to your Services in your cluster. $[prodname] comes with LoadBalancer IPAM deployed as part of $[prodname] Kube-controllers. + +### Before you begin... + +Ensure that you have a cluster with $[prodname] installed, and kube-controllers running. + +### Enable LoadBalancer controller + +By default, LoadBalancer controller is enabled in $[prodname] kube-controllers configuration. You can verify the status by checking the KubeControllersConfiguration resource. + +```bash +kubectl get kubecontrollersconfiguration default -o yaml +``` + +The output should contain LoadBalancer controller configuration as follows: + +```yaml +spec: + controllers: + loadBalancer: + assignIPs: AllServices +``` + +If LoadBalancer controller is not enabled, you can enable it by patching the KubeControllersConfiguration resource: + +```bash +kubectl patch kubecontrollersconfiguration default --type=merge --patch '{"spec": {"controllers":{"loadBalancer":{"assignIPs": "AllServices"}}}}' +``` + +### IP Pool for Service of type LoadBalancer + +$[prodname] does not automatically provide default IP Pool for LoadBalancer IP address management. You will need to create an IP Pool with `allowedUses` LoadBalancer for $[prodname] to start assigning Service IPs. + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: loadbalancer-ip-pool +spec: + cidr: 192.210.0.0/20 + blockSize: 24 + natOutgoing: true + disabled: false + assignmentMode: Automatic + allowedUses: + - LoadBalancer +``` + +:::note + +You can create multiple IP Pools with allowedUses LoadBalancer as long as there are no CIDR conflicts. By setting assignmentMode to Manual you can reserve the IP Pool for manual assignments only. Explore more about [manual assignment](#manual-service-ip-address-assignment) + +::: + +### Automatic Service IP address assignment + +When you create Service of type LoadBalancer $[prodname] kube-controller will automatically detect the new Service and assign an IP address from available IP Pool. If no address is available, the Service will remain in pending state until an address is available. + +```yaml +apiVersion: v1 +kind: Service +metadata: + name: service-loadbalancer +spec: + selector: + app: nginx + ports: + - port: 80 + targetPort: 80 + name: default + type: LoadBalancer +``` + +### Manual Service IP address assignment + +There are cases where you would like to be more specific about what IP address $[prodname] assigns to your Service. With annotations, you can specify how IP address should be assigned. The annotations can be added and removed as needed during the lifetime of the Service. When you remove an annotation $[prodname] kube-controller will check if the assigned IP is still valid and potentially assign a new one. + +#### Specify IP Pool + +##### IPv4 Pool + +Annotate the Service with projectcalico.org/ipv4pools to assign IP address from the selected IP Pools. You can specify multiple IP Pools in this annotation, $[prodname] will pick an available IP address to assign. + +``` +"projectcalico.org/ipv4pools": '["loadBalancerIPv4Pool"]' +``` + +##### IPv6 Pool + +Annotate the Service with projectcalico.org/ipv6pools to assign IP address from the selected IP Pools. You can specify multiple IP Pools in this annotation, $[prodname] will pick an available IP address to assign. + +``` +"projectcalico.org/ipv6pools": '["loadBalancerIPv6Pool"]' +``` + +:::note + +In dual-stack environment you don't have to specify both annotations. If IP Pool was not specified for IP family, $[prodname] will automatically assign available IP address from available IP Pool + +::: + +#### Specifying IP address + +Annotate the Service with projectcalico.org/loadBalancerIPs to assign specific IP address. The address must be available otherwise $[prodname] will not be able to assign the address. Currently you can only specify one IPv4 and one IPv6 address at the same time. + +``` +"projectcalico.org/loadBalancerIPs": '["x.x.x.x"]' +``` + +:::note + +If Service contains "projectcalico.org/loadBalancerIPs" annotation and either "projectcalico.org/ipv6pools" or "projectcalico.org/ipv4pools" are present, $[prodname] will favour projectcalico.org/loadBalancerIPs annotation and try to assign that IP address. There is no fall back to an IP Pool if the specified address is not available. + +::: + +### Manage LoadBalancer kube-controller assignment mode + +In certain cases you might not wish for $[prodname] to automatically assign IP addresses to your Service. This can be useful in case you have multiple Service IPAM solutions in your cluster. +LoadBalancer kube-controller is able to operate in two distinct modes ```AllServices``` in which $[prodname] assigns IP address to each Service in your cluster and ```RequestedServicesOnly``` in which $[prodname] only assigns IP addresses to Service with annotations mentioned above or if `Service.spec.loadBalancerClass` is set to `calico`. +You can change the mode at any point, but note that switching to ```RequestedServicesOnly``` will unassign any addresses from Services that do not contain the above annotations. + +```bash +kubectl patch kubecontrollersconfiguration default --type=merge --patch '{"spec": {"controllers":{"loadBalancer":{"assignIPs": "RequestedServicesOnly"}}}}' +``` + +:::note + +LoadBalancer controller considers the `Service.spec.loadBalancerClass`, it will skip assignment for Services where `loadBalancerClass` is set to a value other than `calico`, regardless of the `assignIPs` setting. + +::: + +### Additional resources + +Calico LoadBalancer IP address management works in conjunction with other $[prodname] components. $[prodname] kube-controllers provides only the IP address management, to advertise LoadBalancer IPs you will have to update your BGP configuration. You can find out more information at: + * [Advertising Kubernetes service IP addresses](../../networking/configuring/advertise-service-ips.mdx#advertise-service-load-balancer-ip-addresses) + * [IP Pool](../../reference/resources/ippool.mdx) + * [Kube-controllers configuration](../../reference/resources/kubecontrollersconfig.mdx#loadbalancercontroller) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/ipam/use-specific-ip.mdx b/calico-cloud_versioned_docs/version-24-1/networking/ipam/use-specific-ip.mdx new file mode 100644 index 0000000000..a2af64ff58 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/ipam/use-specific-ip.mdx @@ -0,0 +1,65 @@ +--- +description: Pin a Kubernetes pod to a chosen address in a Calico Cloud connected cluster by setting a pod annotation that supplies the requested address. +--- + +# Use a specific IP address with a pod + +import DetermineIpam from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_determine-ipam.mdx'; + +## Big picture + +Choose the IP address for a pod instead of allowing $[prodname] to choose automatically. + +## Value + +Some applications require the use of stable IP addresses. Also, you may want to create entries in external DNS servers that point directly to pods, and this requires static IPs. + +## Concepts + +### Kubernetes pod CIDR + +The **Kubernetes pod CIDR** is the range of IPs Kubernetes expects pod IPs to be assigned from. It is defined for the entire cluster and is used by various Kubernetes components to determine whether an IP belongs to a pod. For example, kube-proxy treats traffic differently if an IP is from a pod than if it is not. All pod IPs must be in the CIDR range for Kubernetes to function correctly. + +**IP Pools** + +IP pools are ranges of IP addresses from which $[prodname] assigns pod IPs. Static IPs must be in an IP pool. + +## Before you begin... + +Your cluster must be using Calico IPAM to use this feature. + + + +## How to + +Annotate the pod with cni.projectcalico.org/ipAddrs set to a list of IP addresses to assign, enclosed in brackets. For example: + +``` + "cni.projectcalico.org/ipAddrs": "[\"192.168.0.1\", \"2001:db8::1\"]" +``` + +Note the use of the escaped `\"` for the inner double quotes around the addresses. + +The address must be within a configured $[prodname] IP pool and not currently in use. The annotation must be present when the pod is created; adding it later has no effect. + +Note that currently only a single IP address per IP family is supported per-pod using this annotation. + +### Reserving IPs for manual assignments + +The `cni.projectcalico.org/ipAddrs` annotation requires the IP address to be within an IP pool. This means that, +by default, $[prodname] may decide to use the IP address that you select for another workload or for an internal +tunnel address. To prevent this, there are several options: + +- To reserve a whole IPPool for manual allocations, you can set its node selector in the [IP pool](../../reference/resources/ippool.mdx) to `"!all()"`. Since the `!all()` + cannot match any nodes, the IPPool will not be used for any automatic assignments. + +- To reserve part of a pool, you can create an [IP reservation](../../reference/resources/ipreservation.mdx). This allows for certain IPs to be reserved so + that Calico IPAM will not use them automatically. However, manual assignments (using the annotation) can still use + IPs that are "reserved". + +- To prevent $[prodname] from using IPs from a certain pool for internal IPIP and/or VXLAN tunnel addresses, you + can set the `allowedUses` field on the [IP Pool](../../reference/resources/ippool.mdx) to `["Workload"]`. + +## Additional resources + +For help configuring $[prodname] CNI and $[prodname] IPAM, see [Configuring the $[prodname] CNI Plugins](../../reference/component-resources/configuration.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/networking/training/about-kubernetes-networking.mdx b/calico-cloud_versioned_docs/version-24-1/networking/training/about-kubernetes-networking.mdx new file mode 100644 index 0000000000..96ff895268 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/training/about-kubernetes-networking.mdx @@ -0,0 +1,137 @@ +--- +description: Reference primer for Kubernetes networking concepts that help when operating Calico Cloud — pod IPs, services, DNS, NAT outgoing, and dual stack. +--- + +# Kubernetes network model + +{/* Do not change the canonical_url above *or the title*. Content is shared with Marketing and is used for SEO purposes. If you change the title, it will screw up Marketing metrics. */} + +:::note + +This guide provides educational material that is not specific to $[prodname]. + +::: + +Kubernetes defines a network model that helps provide simplicity and consistency across a range of networking +environments and network implementations. The Kubernetes network model provides the foundation for understanding how +containers, pods, and services within Kubernetes communicate with each other. This guide explains the key concepts and +how they fit together. + +In this guide you will learn: + +- The fundamental network behaviors the Kubernetes network model defines. +- How Kubernetes works with a variety of different network implementations. +- What Kubernetes Services are. +- How DNS works within Kubernetes. +- What "NAT outgoing" is and when you would want to use it. +- What "dual stack" is. + +## The Kubernetes network model + +The Kubernetes network model specifies: + +- Every pod gets its own IP address +- Containers within a pod share the pod IP address and can communicate freely with each other +- Pods can communicate with all other pods in the cluster using pod IP addresses (without + [NAT](about-networking.mdx)) +- Isolation (restricting what each pod can communicate with) is defined using network policies + +As a result, pods can be treated much like VMs or hosts (they all have unique IP addresses), and the containers within +pods very much like processes running within a VM or host (they run in the same network namespace and share an IP +address). This model makes it easier for applications to be migrated from VMs and hosts to pods managed by Kubernetes. +In addition, because isolation is defined using network policies rather than the structure of the network, the network +remains simple to understand. This style of network is sometimes referred to as a "flat network". + +Note that, although very rarely needed, Kubernetes does also support the ability to map host ports through to pods, or +to run pods directly within the host network namespace sharing the host's IP address. + +## Kubernetes network implementations + +Kubernetes built in network support, kubenet, can provide some basic network connectivity. However, it is more common to +use third party network implementations which plug into Kubernetes using the CNI (Container Network Interface) API. + +There are lots of different kinds of CNI plugins, but the two main ones are: + +- network plugins, which are responsible for connecting pod to the network +- IPAM (IP Address Management) plugins, which are responsible for allocating pod IP addresses. + +$[prodname] provides both network and IPAM plugins, but can also integrate and work seamlessly with some other CNI +plugins, including AWS, Azure, and Google network CNI plugins, and the host local IPAM plugin. This flexibility allows +you to choose the best networking options for your specific needs and deployment environment. +{/*//TODO-CC-XREFS You can read more about +this in the $[prodname] [determine best networking option](/networking/determine-best-networking) +guide. +*/} + +## Kubernetes Services + +Kubernetes [Services](https://kubernetes.io/docs/concepts/services-networking/service/) provide a way of abstracting access to a group +of pods as a network service. The group of pods is usually defined using a [label selector](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels) +. Within the cluster the +network service is usually represented as virtual IP address, and kube-proxy load balances connections to the virtual IP +across the group of pods backing the service. The virtual IP is discoverable through Kubernetes DNS. The DNS name and +virtual IP address remain constant for the life time of the service, even though the pods backing the service may be +created or destroyed, and the number of pods backing the service may change over time. + +Kubernetes Services can also define how a service is accessed from outside of the cluster, for example using + +- a node port, where the service can be accessed via a specific port on every node +- or a load balancer, whether a network load balancer provides a virtual IP address that the service can be accessed via + from outside the cluster. + +Note that when using $[prodname] in on-prem deployments you can also [advertise service IP addresses](../configuring/advertise-service-ips.mdx) +, allowing services to be conveniently accessed without +going via a node port or load balancer. + +## Kubernetes DNS + +Each Kubernetes cluster provides a DNS service. Every pod and every service is discoverable through the Kubernetes DNS +service. + +For example: + +- Service: `my-svc.my-namespace.svc.cluster-domain.example` +- Pod: `pod-ip-address.my-namespace.pod.cluster-domain.example` +- Pod created by a deployment exposed as a service: + `pod-ip-address.deployment-name.my-namespace.svc.cluster-domain.example`. + +The DNS service is implemented as Kubernetes Service that maps to one or more DNS server pods (usually CoreDNS), that +are scheduled just like any other pod. Pods in the cluster are configured to use the DNS service, with a DNS search list +that includes the pod's own namespace and the cluster's default domain. + +This means that if there is a service named `foo` in Kubernetes namespace `bar`, then pods in the same namespace can +access the service as `foo`, and pods in other namespaces can access the service as `foo.bar` + +Kubernetes supports a rich set of options for controlling DNS in different scenarios. You can read more about these in +the Kubernetes guide [DNS for Services and Pods](https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/). + +## NAT outgoing + +The Kubernetes network model specifies that pods must be able to communicate with each other directly using pod IP +addresses. But it does not mandate that pod IP addresses are routable beyond the boundaries of the cluster. Many +Kubernetes network implementations use [overlay networks](about-networking.mdx#overlay-networks). +Typically for these deployments, when a pod initiates a connection to an IP address outside of the cluster, the node +hosting the pod will SNAT (Source Network Address Translation) map the source address of the packet from the pod IP to +the node IP. This enables the connection to be routed across the rest of the network to the destination (because the +node IP is routable). Return packets on the connection are automatically mapped back by the node replacing the node IP +with the pod IP before forwarding the packet to the pod. + +When using $[prodname], depending on your environment, you can generally choose whether you prefer to run an +overlay network, or prefer to have fully routable pod IPs. +{/*TODO-XREFS-CC +You can read more about this in the $[prodname] +[determine best networking option](/networking/determine-best-networking) guide. +*/} +$[prodname] also +allows you to [configure outgoing NAT](../configuring/workloads-outside-cluster.mdx) for specific IP address +ranges if more granularity is desired. + +## Dual stack + +If you want to use a mix of IPv4 and IPv6 then you can enable Kubernetes [dual-stack](https://kubernetes.io/docs/concepts/services-networking/dual-stack/) mode. When enabled, all +pods will be assigned both an IPv4 and IPv6 address, and Kubernetes Services can specify whether they should be exposed +as IPv4 or IPv6 addresses. + +## Additional resources + +- [The Kubernetes Network Model](https://kubernetes.io/docs/concepts/cluster-administration/networking/#the-kubernetes-network-model) diff --git a/calico-cloud_versioned_docs/version-24-1/networking/training/about-networking.mdx b/calico-cloud_versioned_docs/version-24-1/networking/training/about-networking.mdx new file mode 100644 index 0000000000..7b1a17e5fd --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/training/about-networking.mdx @@ -0,0 +1,169 @@ +--- +description: Reference primer for networking fundamentals — OSI layers, packet anatomy, MTU, IP addressing, routing, overlays, DNS, and NAT — that underpin Calico Cloud. +--- + +# Networking overview + +:::note + +This guide provides educational material that is not specific to $[prodname]. + +::: + +You can get up and running with $[prodname] by following any of the $[prodname] install +guides without needing to be a networking expert. $[prodname] hides the complexities for +you. However, if you would like to learn more about networking so you can better understand what is happening under the +covers, this guide provides a short introduction to some of the key fundamental networking concepts for anyone who is +not already familiar with them. + +In this guide you will learn: + +- The terms used to described different layers of the network. +- The anatomy of a network packet. +- What MTU is and why it makes a difference. +- How IP addressing, subnets, and IP routing works. +- What an overlay network is. +- What DNS and NAT are. + +## Network layers + +The process of sending and receiving data over a network is commonly categorized into 7 layers (referred to as the [OSI model](https://en.wikipedia.org/wiki/OSI_model)). The layers are +typically abbreviated as L1 - L7. You can think of data as passing through each of these layers in turn as it is sent or +received from an application, with each layer being responsible for a particular part of the processing required to +send or receive the data over the network. + +![OSI network layers diagram](/img/calico-enterprise/osi-network-layers.svg) + +In a modern enterprise or public cloud network, the layers commonly map as follows: + +- L5-7: all the protocols most application developers are familiar with. e.g. HTTP, FTP, SSH, SSL, DNS. +- L4: TCP or UDP, including source and destination ports. +- L3: IP packets and IP routing. +- L2: Ethernet packets and Ethernet switching. + +## Anatomy of a network packet + +When sending data over the network, each layer in the network stack adds its own header containing the control/metadata +the layer needs to process the packet as it traverses the network, passing the resulting packet on to the next +layer of the stack. In this way the complete packet is produced, which includes all the control/metadata required by +every layer of the stack, without any layer understanding the data or needing to process the control/metadata of +adjacent network layers. + +![Anatomy of a network packet](/img/calico-enterprise/anatomy-of-a-packet.svg) + +## IP addressing, subnets and IP routing + +The L3 network layer introduces IP addresses and typically marks the boundary between the part of networking that +application developers care about, and the part of networking that network engineers care about. In particular +application developers typically regard IP addresses as the source and destination of the network traffic, but have much +less of a need to understand L3 routing or anything lower in the network stack, which is more the domain of network +engineers. + +There are two variants of IP addresses: IPv4 and IPv6. + +- IPv4 addresses are 32 bits long and the most commonly used. They are typically represented as 4 bytes in decimal (each + 0-255) separated by dots. e.g. `192.168.27.64`. There are several ranges of IP addresses that are reserved as + "private", that can only be used within local private networks, are not routable across the internet. These can be + reused by enterprises as often as they want to. In contrast "public" IP addresses are globally unique across the whole + of the internet. As the number of network devices and networks connected to the internet has grown, public IPv4 + addresses are now in short supply. +- IPv6 addresses are 128 bits long and designed to overcome the shortage of IPv4 address space. They are typically + represented by 8 groups of 4 digit hexadecimal numbers. e.g. `1203:8fe0:fe80:b897:8990:8a7c:99bf:323d`. Due to the 128 + bit length, there's no shortage of IPv6 addresses. However, many enterprises have been slow to adopt IPv6, so for now + at least, IPv4 remains the default for many enterprise and data center networks. + +Groups of IP addresses are typically represented using [CIDR notation](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) that consists of an IP address and number of +significant bits on the IP address separated by a `/`. For example, `192.168.27.0/24` represents the group of 256 IP +addresses from `192.168.27.0` to `192.168.27.255`. + +A group of IP addresses within a single L2 network is referred to as a subnet. Within a subnet, packets can be sent +between any pair of devices as a single network hop, based solely on the L2 header (and footer). + +To send packets beyond a single subnet requires L3 routing, with each L3 network device (router) being responsible for +making decisions on the path to send the packet based on L3 routing rules. Each network device acting as a router has +routes that determine where a packet for a particular CIDR should be sent next. So for example, in a Linux system, a +route of `10.48.0.128/26 via 10.0.0.12 dev eth0` indicates that packets with destination IP address in `10.48.0.128/26` +should be routed to a next network hop of `10.0.0.12` over the `eth0` interface. + +Routes can be configured statically by an administrator, or programmed dynamically using routing protocols. When using +routing protocols each network device typically needs to be configured to tell it which other network devices it should +be exchanging routes with. The routing protocol then handles programming the right routes across the whole of the +network as devices are added or removed, or network links come in or out of service. + +One common routing protocol used in large enterprise and data center networks is [BGP](https://en.wikipedia.org/wiki/Border_Gateway_Protocol). BGP is one of the main protocols that powers +the internet, so scales incredibly well, and is very widely supported by modern routers. + +## Overlay networks + +An overlay network allows network devices to communicate across an underlying network (referred to as the underlay) +without the underlay network having any knowledge of the devices connected to the overlay network. From the point of +view of the devices connected to the overlay network, it looks just like a normal network. There are many different +kinds of overlay networks that use different protocols to make this happen, but in general they share the same common +characteristic of taking a network packet, referred to as the inner packet, and encapsulating it inside an outer network +packet. In this way the underlay sees the outer packets without needing to understand how to handle the inner packets. + +How the overlay knows where to send packets varies by overlay type and the protocols they use. Similarly exactly how the +packet is wrapped varies between different overlay types. In the case of VXLAN for example, the inner packet is wrapped +and sent as UDP in the outer packet. + +![Anatomy of an overlay network packet](/img/calico-enterprise/anatomy-of-an-overlay-packet.svg) + +Overlay networks have the advantage of having minimal dependencies on the underlying network infrastructure, but have +the downsides of: + +- having a small performance impact compared to non-overlay networking, which you might want to avoid if running + network intensive workloads +- workloads on the overlay are not easily addressable from the rest of the network. so NAT gateways or load balancers + are required to bridge between the overlay and the underlay network for any ingress to, or egress from, the overlay. + +$[prodname] networking options are exceptionally flexible, so in general you can choose whether you prefer +$[prodname] to provide an overlay network, or non-overlay network. +{/*TODO-XREFS-CC +You can read more about this in the $[prodname] +[determine best networking option](/networking/determine-best-networking) guide. +*/} + +## DNS + +While the underlying network packet flow across the network is determined using IP addresses, users and applications +typically want to use well known names to identify network destinations that remain consistent over time, even if the +underlying IP addresses change. For example, to map `google.com` to `216.58.210.46`. This translation from name to IP +address is handled by [DNS](https://en.wikipedia.org/wiki/Domain_Name_System). DNS runs on top of the base networking described so far. Each device connected to a network is typically configured +with the IP addresses of one or more DNS servers. When an application wants to connect to a domain name, a DNS message is +sent to the DNS server, which then responds with information about which IP address(es) the domain name maps to. The +application can then initiate its connection to the chosen IP address. + +## NAT + +Network Address Translation ([NAT](https://en.wikipedia.org/wiki/Network_address_translation)) is the process of mapping an IP address in a packet +to a different IP address as the packet passes through the device performing the NAT. Depending on the use case, NAT can +apply to the source or destination IP address, or to both addresses. + +One common use case for NAT is to allow devices with private IP address to talk to devices with public IP address across +the internet. For example, if a device with a private IP address attempts to connect to a public IP address, then the +router at the border of the private network will typically use SNAT (Source Network Address Translation) to map the +private source IP address of the packet to the router's own public IP address before forwarding it on to the internet. +The router then maps response packets coming in the opposite direction back to the original private IP address, so +packets flow end-to-end in both directions, with neither source or destination being aware the mapping is happening. The +same technique is commonly used to allow devices connected to an overlay network to connect with devices outside of the +overlay network. + +Another common use case for NAT is load balancing. In this case the load balancer performs DNAT (Destination Network +Address Translation) to change the destination IP address of the incoming connection to the IP address of the chosen +device it is load balancing to. The load balancer then reverses this NAT on response packets so neither source or +destination device is aware the mapping is happening. + +## MTU + +The Maximum Transmission Unit ([MTU](https://en.wikipedia.org/wiki/Maximum_transmission_unit)) of a network link is the maximum size of packet that +can be sent across that network link. It is common for all links in a network to be configured with the same MTU to +reduce the need to fragment packets as they traverse the network, which can significantly lower the performance of the +network. In addition, TCP tries to learn path MTUs, and adjust packet sizes for each network path based on the smallest +MTU of any of the links in the network path. When an application tries to send more data than can fit in a single +packet, TCP will fragment the data into multiple TCP segments, so the MTU is not exceeded. + +Most networks have links with an MTU of 1,500 bytes, but some networks support MTUs of 9,000 bytes. In a Linux system, +larger MTU sizes can result in lower CPU being used by the Linux networking stack when sending large amounts of data, +because it has to process fewer packets for the same amount of data. Depending on the network interface hardware being +used, some of this overhead may be offloaded to the network interface hardware, so the impact of small vs large MTU +sizes varies from device to device. diff --git a/calico-cloud_versioned_docs/version-24-1/networking/training/index.mdx b/calico-cloud_versioned_docs/version-24-1/networking/training/index.mdx new file mode 100644 index 0000000000..b5ce06d44b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/networking/training/index.mdx @@ -0,0 +1,11 @@ +--- +description: Networking training material for Calico Cloud — fundamentals of network layers, IP addressing, routing, and the Kubernetes network model. +hide_table_of_contents: true +--- + +# Networking basics + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/observability/alerts.mdx b/calico-cloud_versioned_docs/version-24-1/observability/alerts.mdx new file mode 100644 index 0000000000..82926de69e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/alerts.mdx @@ -0,0 +1,234 @@ +--- +description: Configure alerts and review alert events for Calico Cloud features in the web console or CLI. Use built-in templates for visibility and security signals. +--- + +# Manage alerts + +## Big picture + +Manage alerts and alert events for $[prodname] features. + +## Value + +You can configure alerts for many $[prodname] features. Alerts are critical to teams for different reasons, for example: + +- **Visibility and troubleshooting** - alerts may indicate infrastructure problems, application bugs, or performance degradation +- **Security** - alerts on suspicious traffic or workload behavior may indicate a compromise or malicious actor + +You can manage alerts and alert events in the web console, or using the CLI. $[prodname] also provides alert templates +for common tasks that you can rename and edit to suit your own needs. + +## Before you begin + +**Recommended** + +We recommend turning down the aggregation level for flow logs to ensure that you see pod-specific results. $[prodname] aggregates flow logs over the external IPs for allowed traffic, and alert events will not provide pod-specific results (unless the traffic is denied by policy). + +:::caution + +Turning down aggregation levels for flow logs increases the amount of log data generated and may increase your $[prodname] bill. + +::: + +To turn down aggregation on flow logs, go to [FelixConfiguration](../reference/resources/felixconfig.mdx) and set the field, **flowLogsFileAggregationKindForAllowed** to **1**. + +## How to + +- [Manage alerts in the web console](#manage-alerts-in-the-web-console) +- [Manage alerts using CLI](#manage-alerts-using-cli) + +### Manage alerts in the web console + +You can view alert events in the web console in several places: the **Alerts** page and **Service Graph**. + +Click **Activity**, **Alerts** to follow along. + +**Alerts page** + +The Alerts page lists **alert events** that are generated by alerts that you’ve configured. (A list of Alerts can be found by clicking the **Alert configuration** icon). + +![alerts-list](/img/calico-enterprise/alerts-list.png) + +You can create alerts for many $[prodname] features. Although the following list of features is not exhaustive and will grow, you get a sense of the range of alerts that can be displayed on this page. + +- $[prodname] logs from Elasticsearch (flow, dns, audit, bgp, L7) +- Deep packet inspection (DPI) +- Threat defense (suspicious IPs, suspicious domains) +- Web Application Firewall (WAF) + +Note the following: + +- The alert event list will be empty, if no alerts have occurred yet +- You can dismiss alert events from view using the checkboxes or bulk action +- The list may contain alert events that are identical or nearly identical. For nearly identical events, you can see differences in the `record` field when you expand the event. +- Because alert events share the same interface, fields that do not apply to the alert are noted by “N/A” +- You can filter alert events by Type. + + ![filter-alerts](/img/calico-enterprise/filter-alerts.png) + + Note these types: + + - **Custom** - filters legacy global alert events that were created before v3.12 + - **Global Alert** - includes alerts for $[prodname] Elasticsearch logs (audit, dns, flow, L7, WAF) + +**Add/edit/delete alerts** + +To manage alerts, click the **Alerts Configuration** icon. + +The following alert is an example of a global alert in the list view. This sample alert generates alert events when there are 100 flows in the cluster in the last 5 mins. (The YAML version of this alert is shown in the section on using the [CLI](#examples).) + +![alert-list-view](/img/calico-enterprise/alert-list-view.png) + +To create a new alert, click the **New** drop-down menu, and select **Blank**. + +Global alerts use a domain-specific query language to select records from a data set to use in the alert. You can also select/omit specific namespaces. + +![alert-example-ui](/img/calico-enterprise/alert-example-ui.png) + +For help with fields on this page, see [GlobalAlert](../reference/resources/globalalert.mdx). + +**Alert templates** + +From the **New** drop-down menu, select **Template**. + +![alert-template](/img/calico-enterprise/alert-template.png) + +The template list contains alerts for common tasks created by $[prodname]. With templates you can: + +- Update and rename an existing template +- Create a new template from scratch +- Create a new alert and save it as a template + +### Manage alerts using CLI + +This section provides examples of how to create and delete global alerts using `kubectl` and YAML files. + +**Create a global alert** + +1. Create a YAML file with one or more alerts. +1. Apply the alert to your cluster. + + ```bash + kubectl apply -f + ``` + +1. Wait until the alert runs and check the status. + + ```bash + kubectl get globalalert -o yaml + ``` + +1. In the web console, go to the **Alerts** page to view alert events. + +### Examples + +The following alert generates alert events when there are 100 flows in the cluster in the last 5 mins. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalAlert +metadata: + name: example-flows +spec: + description: '100 flows Example' + summary: 'Flows example ${count} > 100' + severity: 100 + dataSet: flows + metric: count + condition: gt + threshold: 100 +``` + +The following alert generates alert events when there is ssh traffic in the default namespace. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalAlert +metadata: + name: network.ssh +spec: + description: 'ssh flows to default namespace' + summary: '[flows] ssh flow in default namespace detected from ${source_namespace}/${source_name_aggr}' + severity: 100 + period: 10m + lookback: 10m + dataSet: flows + query: proto='tcp' AND action='allow' AND dest_port='22' AND (source_namespace='default' OR dest_namespace='default') AND reporter=src + aggregateBy: [source_namespace, source_name_aggr] + field: num_flows + metric: sum + condition: gt + threshold: 0 +``` + +The following alert generates alert events when $[prodname] globalnetworksets are modified. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalAlert +metadata: + name: policy.globalnetworkset +spec: + description: 'Changed globalnetworkset' + summary: '[audit] [privileged access] change detected for ${objectRef.resource} ${objectRef.name}' + severity: 100 + period: 10m + lookback: 10m + dataSet: audit + query: (verb=create OR verb=update OR verb=delete OR verb=patch) AND "objectRef.resource"=globalnetworksets + aggregateBy: [objectRef.resource, objectRef.name] + metric: count + condition: gt + threshold: 0 +``` + +The following alert generates alert events for all flow from processes in the data set. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalAlert +metadata: + name: example-process-set-embedded +spec: + description: Generate alerts for all flows from processes in the set + summary: Generate alerts for all flows from processes in the set + severity: 100 + dataSet: flows + query: process_name IN {"python?", "*checkoutservice"} +``` + +The following example generates alert events for DNS lookups that are not in the allowed domain set. Because this set can be potentially large, a variable is used in the query string and is referenced in the substitutions list. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalAlert +metadata: + name: example-domain-set-variable +spec: + description: Generate alerts for all DNS lookups not in the domain set + summary: Generate alerts for all DNS lookups not in the domain set with variable + severity: 100 + dataSet: dns + query: qname NOTIN ${domains} + substitutions: + - name: domains + values: + - '*cluster.local' + - '?.mydomain.com' +``` + +**Delete a global alert** + +To delete a global alert and stop all alert event generation, use the following command. + +```bash +kubectl delete globalalert +``` + +## Additional resources + +- [GlobalAlert and templates](../reference/resources/globalalert.mdx) +- Alerts for [Deep packet inspection](../threat/deeppacketinspection.mdx) +- Alerts for [suspicious IPs](../threat/suspicious-ips.mdx) +- Alerts for [suspicious domains](../threat/suspicious-domains.mdx) +- Alerts for [Web Application Firewall](../threat/web-application-firewall.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/observability/create-custom-dashboard.mdx b/calico-cloud_versioned_docs/version-24-1/observability/create-custom-dashboard.mdx new file mode 100644 index 0000000000..6442d20491 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/create-custom-dashboard.mdx @@ -0,0 +1,72 @@ +--- +description: Build custom dashboards in the Calico Cloud web console by importing cards from standard dashboards or constructing new cards from flow, DNS, and L7 fields. +--- + +import Screenshot from '/src/___new___/components/Screenshot' +import IconDashboard from '/img/icons/dashboard.svg'; +import IconEllipsis from '/img/icons/ellipsis.svg'; +import IconGripLines from '/img/icons/grip-lines.svg'; + +# Create a custom dashboard + +:::note + +Creating custom dashboards is a tech preview feature. +Tech preview features are subject to significant changes before they become GA. + +::: + +You can customize your dashboard views by importing and modifying cards from the standard dashboards. +You can also build cards from scratch using any fields from your flow, DNS, and L7 logs. + + + +## Prerequisites + +* You are signed in to the web console as a user with the Owner, Admin, or Dashboards Admin role. + +## Create a new dashboard + +You can create your own custom dashboard by importing cards from other standard and custom dashboards. + +1. From the web console, click **Dashboards > New Dashboard**. +1. Enter a name for your custom dashboard, and then press **Enter**. +1. To import cards from existing dashboards, click **Import Cards**. + In the dialog that appears, select the cards you want, and then click **Import**. + The page now displays the cards you selected. +1. Add, delete, or move the cards to create the arrangement that suits your needs. + * To add a card, click **Actions > Import**, and then select a card. + * To delete a card, click the ellipsis icon on the card, and then click **Delete**. + * To export data from a card as a CSV file, click the ellipsis icon on the card, and then click **Export**. + +## Modify an existing card + + + +You can quickly edit any card on a custom dashboard to display the information you need. +Even if you’re not creating a card from scratch, you can adjust queries, filters, or visual settings to refine the data. + +1. From a custom dashboard, locate the card you want to modify. + On the card, click the ellipsis icon , and then click **Edit**. +1. On the **Edit Card** screen, you can change the title, add a filter, change the chart type, or make any other modifications you want. + The live chart view is updated in real time. +1. When you have finished making changes, click **Save**. + The modified chart now appears on your custom dashboard page. + +## Create a new card + +You can create a card by defining a query and selecting how the results are displayed. +This lets you create custom data visualizations based on your specific needs. + +***Prerequisites*** + +* You have a working knowledge of databases, search queries, and structured data analysis. + +***Procedure*** + +1. From a custom dashboard page, go to **Actions > Add New Card**. +1. Build your card selecting a data collection and chart type. + Depending on the chart type, you'll be prompted to specify fields for the **Group By** and **Aggregation** sections. +1. When you have finished making changes, click **Save**. + The modified chart now appears on your custom dashboard page. + diff --git a/calico-cloud_versioned_docs/version-24-1/observability/dashboards.mdx b/calico-cloud_versioned_docs/version-24-1/observability/dashboards.mdx new file mode 100644 index 0000000000..d5fe313ef9 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/dashboards.mdx @@ -0,0 +1,86 @@ +--- +description: Visualize connected cluster activity in the Calico Cloud web console with prebuilt and custom dashboards for cluster health, traffic volume, DNS, flow logs, and HTTP traffic. +--- + +import Screenshot from '/src/___new___/components/Screenshot' +import IconDashboard from '/img/icons/dashboard.svg'; +import IconEllipsis from '/img/icons/ellipsis.svg'; +import IconGripLines from '/img/icons/grip-lines.svg'; + +# Dashboards + +Dashboards help you see what's going on in your cluster. +See how your cluster is performing and visualize your system's log data. + +## About dashboards + +$[prodname] provides a set of dashboards to help you understand the activity in your cluster. +Each dashboard is made up of graphs, charts, and diagrams that visually represent the data in your logs. + +To view your dashboards, sign in to the web console and click the **Dashboards** icon. + +You can also create your own arrangement by creating a custom dashboard. +With a custom dashboard, you can combine and arrange cards from any of the other dashboards. + +Creating custom dashboards is limited to users with Owner, Admin, or Dashboards Admin permissions. + +## Role access to dashboards + +The following user roles have access to all information in an organization's dashboards: + +* Owner +* Admin +* Dashboards Admin +* Devops +* Security +* Viewer + +Users must be assigned one of the standard roles above, or a custom role with dashboard permissions, to access dashboards. +Administrators can limit dashboard access for custom roles, for example to specific clusters or namespaces in a cluster. + +### Cluster Health + +The **Cluster Health** dashboard provides a birds-eye view of cluster activity. + + + +### Traffic Volume + +The **Traffic Volume** dashboard provides a high-level view of traffic in your cluster. + + + +### DNS Logs + +The **DNS Logs** dashboard summarizes DNS data and logs into metrics, providing high-level information on the types of DNS lookups made, responses, and overall DNS performance. + + + +### Flow Logs + +The **Flow Logs** dashboard gives you an overview of how packets are being sent and received by all the pods in your cluster. +Seeing this data helps you spot unusual flow activity, which may indicate a compromise. + + + +### HTTP Traffic + +The **HTTP Traffic** dashboard provides application performance metrics for in-scope Kubernetes services. +The data can assist service owners and platform personnel in assessing the health of cluster workloads without the need for a full service mesh. +[L7 logs](elastic/l7/overview.mdx) are not enabled by default, and must be configured. + + + +### Network Performance + +The **Network Performance** dashboard provides TCP metrics to help you identify bottlenecks, packet loss, and performance issues. +[Additional TCP statistics](elastic/flow/tcpstats.mdx) such as Round Trip Time, Retransmission and Packet Loss are not enabled by default, and must be configured. + + + +### Ingress Gateway L7 + +The **Ingress Gateway L7** dashboard gives you a clear view of how your gateways are handling traffic and if they are set up correctly. +It helps you track request speeds, find errors quickly, and make sure your apps stay connected and running well. + + \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/aggregation.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/aggregation.mdx new file mode 100644 index 0000000000..74960f6d67 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/aggregation.mdx @@ -0,0 +1,26 @@ +--- +description: Understand how Calico Cloud aggregates log entries by selecting which fields participate in the aggregation key. +--- + +import AggregationKeyDemo from '@site/calico-cloud_versioned_docs/version-24-1/_includes/components/AggregationKeyDemo'; + +# Log aggregation + +## Big picture + +$[prodname] aggregates log entries before writing them to disk. Rather than emitting one record per event, it merges events that share the same values across a chosen set of fields — the *aggregation key* — into a single entry that carries a `count` of how many events were merged. + +Aggregation trades detail for volume: + +- **Including a field** makes the log more granular: more entries, fewer events per entry. +- **Excluding a field** collapses events that differ only in that field into one entry and increments its `count` for each merged event. + +The same mechanism applies across $[prodname] log types (flow, DNS, L7); only the available fields differ. The example below uses L7 (HTTP) traffic to make the behavior concrete. + +## Aggregation playground + + + +## Aggregation fields + +The fields that control the aggregation key, including their accepted values and defaults, are documented in the [Felix Configuration reference](../../reference/component-resources/node/felix/configuration.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/archive-storage.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/archive-storage.mdx new file mode 100644 index 0000000000..1bce0898b9 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/archive-storage.mdx @@ -0,0 +1,211 @@ +--- +description: Forward Calico Cloud flow, DNS, audit, and L7 logs to Syslog, Splunk, or Amazon S3 to retain compliance data beyond managed retention windows. +--- + +# Archive logs + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## Big picture + +Archive $[prodname] logs to SIEMs like Syslog, Splunk, or Amazon S3 to meet compliance storage requirements. + +## Value + +Archiving your $[prodname] Elasticsearch logs to storage services like Amazon S3, Syslog, or Splunk are reliable +options for maintaining and consolidating your compliance data long term. + +## Before you begin + +**Supported logs for export** + +- Syslog - flow, dns, idsevents, audit +- Amazon S3 - l7, flow, dns, runtime, audit +- Splunk - flow, audit, dns + +## How to + +:::note +Because $[prodname] and Kubernetes logs are integral to $[prodname] diagnostics, there is no mechanism to tune down the verbosity. To manage log verbosity, filter logs using your SIEM. +::: + + + + +1. Create an AWS bucket to store your logs. + You will need the bucket name, region, key, secret key, and the path in the following steps. + +2. Create a Secret in the `tigera-operator` namespace named `log-collector-s3-credentials` with the fields `key-id` and `key-secret`. + Example: + + ``` + kubectl create secret generic log-collector-s3-credentials \ + --from-literal=key-id= \ + --from-literal=key-secret= \ + -n tigera-operator + ``` + +3. Update the [LogCollector](../../reference/installation/api.mdx#logcollector) + resource named, `tigera-secure` to include an [S3 section](../../reference/installation/api.mdx#s3storespec) + with your information noted from above. + Example: + + ```yaml + apiVersion: operator.tigera.io/v1 + kind: LogCollector + metadata: + name: tigera-secure + spec: + additionalStores: + s3: + bucketName: + bucketPath: + region: + ``` + + This can be done during installation by editing the custom-resources.yaml + by applying it, or after installation by editing the resource with the command: + + ```bash + kubectl edit logcollector tigera-secure + ``` + + + + +1. Update the [LogCollector](../../reference/installation/api.mdx#logcollector) + resource named `tigera-secure` to include a [Syslog section](../../reference/installation/api.mdx#syslogstorespec) + with your syslog information. + Example: + ```yaml + apiVersion: operator.tigera.io/v1 + kind: LogCollector + metadata: + name: tigera-secure + spec: + additionalStores: + syslog: + # (Required) Syslog endpoint, in the format protocol://host:port + endpoint: tcp://1.2.3.4:514 + # (Optional) If messages are being truncated set this field + packetSize: 1024 + # (Required) Types of logs to forward to Syslog (must specify at least one option) + logTypes: + - Audit + - DNS + - Flows + - IDSEvents + ``` + This can be done during installation by editing the custom-resources.yaml by applying it or after installation by editing the resource with the command: + ```bash + kubectl edit logcollector tigera-secure + ``` +2. You can control which types of $[prodname] log data you would like to send to syslog. + The [Syslog section](../../reference/installation/api.mdx#syslogstorespec) + contains a field called `logTypes` which allows you to list which log types you would like to include. + The allowable log types are: + + - Audit + - DNS + - Flows + - IDSEvents + + Refer to the [Syslog section](../../reference/installation/api.mdx#syslogstorespec) for more details on what data each log type represents. + + :::note + + The log type `IDSEvents` is only supported for a cluster that has log storage configured. + Intrusion detection event data is pulled from log storage directly. + + ::: + + The `logTypes` field is a required, which means you must specify at least one type of log to export to syslog. + +**TLS configuration** + +3. You can enable TLS option for syslog forwarding by including the "encryption" option in the [Syslog section](../../reference/installation/api.mdx#syslogstorespec). + + ```yaml + apiVersion: operator.tigera.io/v1 + kind: LogCollector + metadata: + name: tigera-secure + spec: + additionalStores: + syslog: + # (Required) Syslog endpoint, in the format protocol://host:port + endpoint: tcp://1.2.3.4:514 + # (Optional) If messages are being truncated set this field + packetSize: 1024 + # (Optional) To Configure TLS mode + encryption: TLS + # (Required) Types of logs to forward to Syslog (must specify at least one option) + logTypes: + - Audit + - DNS + - Flows + - IDSEvents + ``` + +4. Using the self-signed CA with the field name tls.crt, create a configmap in the tigera-operator namespace named, syslog-ca. Example: + + :::note + + Skip this step if publicCA bundle is good enough to verify the server certificates. + + ::: + + ```bash + kubectl create configmap syslog-ca --from-file=tls.crt -n tigera-operator + ``` + + + + +**Support** + +In this release, only [Splunk Enterprise](https://www.splunk.com/en_us/products/splunk-enterprise.html) is supported. + +$[prodname] uses Splunk's **HTTP Event Collector** to send data to Splunk server. To copy the flow, audit, and dns logs to Splunk, follow these steps: + +1. Create a HTTP Event Collector token by following the steps listed in Splunk's documentation for your specific Splunk version. Here is the link to do this for [Splunk version 8.0.0](https://docs.splunk.com/Documentation/Splunk/8.0.0/Data/UsetheHTTPEventCollector). + +2. Create a Secret in the `tigera-operator` namespace named `logcollector-splunk-credentials` with the field `token`. + Example: + + ``` + kubectl create secret generic logcollector-splunk-credentials \ + --from-literal=token= \ + -n tigera-operator + ``` + +3. Update the + [LogCollector](../../reference/installation/api.mdx#logcollector) + resource named `tigera-secure` to include + a [Splunk section](../../reference/installation/api.mdx#splunkstorespec) + with your Splunk information. + Example: + + ```yaml + apiVersion: operator.tigera.io/v1 + kind: LogCollector + metadata: + name: tigera-secure + spec: + additionalStores: + splunk: + # Splunk HTTP Event Collector endpoint, in the format protocol://host:port + endpoint: https://1.2.3.4:8088 + ``` + + This can be done during installation by editing the custom-resources.yaml + by applying it or after installation by editing the resource with the command: + + ``` + kubectl edit logcollector tigera-secure + ``` + + + + diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/audit-overview.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/audit-overview.mdx new file mode 100644 index 0000000000..2104f65b73 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/audit-overview.mdx @@ -0,0 +1,37 @@ +--- +description: Calico Cloud audit logs record changes to network policies, tiers, network sets, host endpoints, and other resources across connected clusters. +--- + +# Audit logs + +## Big picture + +$[prodname] audit logs provide security teams and auditors historical data of all changes to resources over time. + +## Concepts + +### Resources used in audit logs + +$[prodname] audit logs are enabled by default for the following resources: + +- Global networkpolicies +- Network policies +- Staged global networkpolicies +- Staged networkpolicies +- Staged Kubernetes network policies +- Global network sets +- Network sets +- Tiers +- Host endpoints + +### Audit logs in the web console + +$[prodname] audit logs are displayed in the Timeline dashboard in the web console. You can filter logs, and export data in .json or .yaml formats. + +![audit-logs](/img/calico-enterprise/audit-logs.png) + +## Required next step + +**Kubernetes resources** are also used in audit-related features, but they are not enabled by default. You must enable Kubernetes resources through the Kubernetes API server. If you miss this step, audit trails will not provide a complete view to your security team. + +- [Enable Kubernetes audit logs](../kube-audit.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/bgp.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/bgp.mdx new file mode 100644 index 0000000000..9d8e8f8942 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/bgp.mdx @@ -0,0 +1,22 @@ +--- +description: Reference of key/value fields in Calico Cloud BGP activity logs in Elasticsearch, with sample queries for IPv4, IPv6, and per-node lookups. +--- + +# BGP logs + +$[prodname] pushes BGP activity logs to Elasticsearch. To view them, go to the Discovery view, and from the dropdown menu, select `tigera_secure_ee_bgp.*` to view the collected BIRD and BIRD6 logs. + +The following table details key/value pairs for constructing queries, including their Elasticsearch datatype. + +| Name | Datatype | Description | +| ------------ | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `logtime` | date | When the log was collected in UTC timestamp format. | +| `host` | keyword | The name of the node where log was collected. | +| `ip_version` | keyword | Contains one of the following values:
● IPv4: Log from BIRD process
● IPv6: Log from BIRD6 process | +| `message` | text | The message contained in the log. | + +Once a set of BGP logs has accumulated in Elasticsearch, you can perform many interesting queries. Depending on the field that you want to query, different techniques are required. For example: + +- To view BGP logs only for IPv4 or IPv6, query on the `ip_version` field and sort by `logtime` +- To see all logs from a specific node, query on the `host` field +- To view events in the cluster, query on the `message` field diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/dns/dns-logs.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/dns/dns-logs.mdx new file mode 100644 index 0000000000..3c49b15863 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/dns/dns-logs.mdx @@ -0,0 +1,64 @@ +--- +description: Reference of key/value fields in Calico Cloud DNS activity logs in Elasticsearch, with guidance for building client and query lookups. +--- + +# Query DNS logs + +$[prodname] pushes DNS activity logs to Elasticsearch, for DNS information that is obtained from [trusted DNS servers](../../../network-policy/domain-based-policy.mdx#trusted-dns-servers). The following table +details the key/value pairs in the JSON blob, including their +[Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html). +This information should assist you in constructing queries. + +| Name | Datatype | Description | +| ------------------ | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `start_time` | date | When the collection of the log began in Unix timestamp format. | +| `end_time` | date | When the collection of the log concluded in Unix timestamp format. | +| `type` | keyword | This field contains one of the following values:
● LOG: Indicates that this is a normal DNS activity log.
● UNLOGGED: Indicates that this log is reporting DNS activity that could not be logged in detail because of [DNSLogsFilePerNodeLimit](../../../reference/resources/felixconfig.mdx#spec). | +| `count` | long | When `type` is:
● LOG: How many DNS lookups there were, during the log collection interval, with details matching this log.
● UNLOGGED: The number of DNS responses that could not be logged in detail because of [DNSLogsFilePerNodeLimit](../../../reference/resources/felixconfig.mdx#spec). In this case none of the following fields are provided. | +| `client_ip` | ip | The IP address of the client pod. A null value indicates aggregation. | +| `client_name` | keyword |

This field contains one of the following values:
● The name of the client pod.
● -: the name of the pod was aggregated. Check client_name_aggr for the pod name prefix.

| +| `client_name_aggr` | keyword | The aggregated name of the client pod. | +| `client_namespace` | keyword | Namespace of the client pod. | +| `client_labels` | array of keywords | Labels applied to the client pod. With aggregation, the label name/value pairs that are common to all aggregated clients. | +| `qname` | keyword | The domain name that was looked up. | +| `qtype` | keyword | The type of the DNS query (e.g. A, AAAA). | +| `qclass` | keyword | The class of the DNS query (e.g. IN). | +| `rcode` | keyword | The result code of the DNS query response (e.g. NoError, NXDomain). | +| `rrsets` | nested | Detailed DNS query response data - see below. | +| `servers` | nested | Details of the DNS servers that provided this response. | +| `latency_count` | long | The number of lookups for which latency was measured. (The same as `count` above, unless some DNS requests were missed, or latency reporting is disabled - see `dnsLogsLatency` in the [FelixConfiguration resource](../../../reference/resources/felixconfig.mdx).) | +| `latency_mean` | long | Mean latency, in nanoseconds. | +| `latency_max` | long | Max latency, in nanoseconds. | + +Each nested `rrsets` object contains response data for a particular name and a particular type and +class of response information. Its key/value pairs are as follows. + +| Name | Datatype | Description | +| ------- | ----------------- | ----------------------------------------------------------------------------------------------------------------------- | +| `name` | keyword | The domain name that this information is for. | +| `type` | keyword | The type of the information (e.g. A, AAAA). | +| `class` | keyword | The class of the information (e.g. IN). | +| `rdata` | array of keywords | Array of data, for the name, of that type and class. For example, when `type` is A, this is an array of IPs for `name`. | + +Each nested `servers` object provides details of a DNS server that provided the information in the +containing log. Its key/value pairs are as follows. + +| Name | Datatype | Description | +| ----------- | ----------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ip` | ip | The IP address of the DNS server. | +| `name` | keyword |

This field contains one of the following values:
● The name of the DNS server pod.
● -: the DNS server is not a pod.

| +| `name_aggr` | keyword |

This field contains one of the following values:
● The aggregated name of the DNS server pod.
● pvt: the DNS server is not a pod. Its IP address belongs to a private subnet.
● pub: the DNS server is not a pod. Its IP address does not belong to a private subnet. It is probably on the public internet.

| +| `namespace` | keyword | Namespace of the DNS server pod, or `-` if the DNS server is not a pod. | +| `labels` | array of keywords | Labels applied to the DNS server pod or host endpoint; empty if there are no labels or the DNS server is not a pod or host endpoint. | + +The `latency_*` fields provide information about the latency of the DNS lookups that contributed to +this log. For each successful DNS lookup $[prodname] measures the time between when the DNS +request was sent and when the corresponding DNS response was received. + +## Query DNS log fields + +After a set of DNS logs has accumulated in Elasticsearch, you can perform many interesting queries. For example, if you query on: + +- `qname`, you can find all of the DNS response information that was provided to clients trying to resolve a particular domain name + +- `rrsets.rdata`, you can find all of the DNS lookups that included a particular IP address in their response data. diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/dns/filtering-dns.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/dns/filtering-dns.mdx new file mode 100644 index 0000000000..6705170172 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/dns/filtering-dns.mdx @@ -0,0 +1,63 @@ +--- +description: Suppress low-value Calico Cloud DNS log entries with Fluentd filters configured through a ConfigMap in the operator namespace of connected clusters. +--- + +# Filter DNS logs + +$[prodname] supports filtering out DNS logs based on user provided +configuration. Use filtering to suppress logs of low significance. + +## Configure DNS filtering + +DNS log filtering is configured through a ConfigMap in the `tigera-operator` +namespace. + +To enable DNS log filtering, follow these steps: + +1. Create a `filters` directory with a file named `dns` with the contents of + your desired filter using [Filter configuration files](#filter-configuration-files). + If you are also adding [flow filters](../flow/filtering.mdx) also add the `flow` file + to the directory. +1. Create the `fluentd-filters` ConfigMap in the `tigera-operator` namespace + with the following command. + ```bash + kubectl create configmap fluentd-filters -n tigera-operator --from-file=filters + ``` + +## Filter configuration files + +The filters defined by the ConfigMap are inserted into the fluentd configuration file. +The [upstream fluentd documentation](https://docs.fluentd.org/filter/grep) +describes how to write fluentd filters. The [DNS log schema](dns-logs.mdx) can be referred to +for the specification of the various fields you can filter based on. Remember to ensure +that the config file is properly indented in the ConfigMap. + +## Example 1: filter out cluster-internal lookups + +This example filters out lookups for domain names ending with ".cluster.local". More +logs could be filtered by adjusting the regular expression "pattern", or by adding +additional `exclude` blocks. + +``` + + @type grep + + key qname + pattern /\.cluster\.local$/ + + +``` + +## Example 2: keep logs only for particular domain names + +This example will filter out all logs _except_ those for domain names ending `.co.uk`. + +``` + + @type grep + + key qname + pattern /\.co\.uk$/ + + +``` diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/dns/index.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/dns/index.mdx new file mode 100644 index 0000000000..82edc1e43e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/dns/index.mdx @@ -0,0 +1,11 @@ +--- +description: Configure and filter DNS activity logs for Calico Cloud. Review the schema, build Elasticsearch queries, and trim low-value entries. +hide_table_of_contents: true +--- + +# Manage DNS logs for Calico Cloud + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/aggregation.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/aggregation.mdx new file mode 100644 index 0000000000..49562ab6e4 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/aggregation.mdx @@ -0,0 +1,123 @@ +--- +description: Tune Calico Cloud flow log aggregation levels to balance managed Elasticsearch volume and cost against pod and IP visibility for allowed and denied traffic. +--- + +# Configure flow log aggregation + +## Big picture + +Configure flow log aggregation level to reduce log volume and costs. + +## Value + +Beyond using filtering to suppress flow logs, $[prodname] provides controls to aggregate flow logs. Although aggressive aggregation levels reduce +flow volume and costs, it can also reduce visibility into specific metadata of allowed and denied traffic. Review this article to see which level of +aggregation is suitable for your implementation. + +## Concepts + +### Volume and cost versus visibility + +$[prodname] enables flow log aggregation by default using default aggregation levels that balance log volume with comprehensive visibility. The +defaults assume that most users do not need to see pod IP information (due to the ephemeral nature of pod IP address allocation) for allowed traffic, +but provides additional details on denied traffic that is more likely to need investigation. However, it all depends on your deployment; +we recommend reviewing aggregation levels to understand what information gets grouped (and thus suppressed from view). + +### Aggregation levels + +For allowed flows, the default aggregation level is 2, and for denied flows the default aggregation level is 1. + +#### Level 0: no aggregation + +Create separate flow logs for each distinct 5-tuple (protocol, source and destination IP and port) observed. This level is not recommended +due to high log volumes. + +#### Level 1: aggregate source ports + +Aggregate flow data relating to multiple connections that only differ in source port. This reduces log volume by discarding source port information, +which is usually ephemeral and of little value. + +#### Level 2: aggregate IPs and source ports + +In addition to the above, aggregate flows that have related sources and destinations into a single log depending on the source and +destination type. +- Pods created by the same pod controller (Deployments, ReplicaSets, etc.) are combined and identified by their common pod prefix, +- IP addresses in the same NetworkSet are aggregated under that shared NetworkSet, +- When no more precise identity is known, arbitrary IP addresses are aggregated to either public (`pub`) and private (`pvt`) as defined in RFC1918. + +#### Level 3: aggregate IPs, source and dest ports + +In addition to the above, aggregate flows with different destination ports that otherwise share together. +This is intended to reduce log volumes in situations where a lot of network probing is expected (for example by `nmap`) but is not typically needed +unless log analysis indicates it will be beneficial. + +### Understanding aggregation level differences + +Here are examples of pod-to-pod flows, highlighting the differences between flow logs at various aggregation levels. + +By suppressing the source port, aggregation level 1 minimizes the flow logs generated for applications that make many connections to the same destination. +The two flows originating from `client-a` without aggregation are combined into one. + +In Kubernetes, pod controllers (Deployments, ReplicaSets, etc.) automatically name the pods they create with a common prefix. +For example, the pods `nginx-1` and `nginx-2` are created by the ReplicaSet `nginx`. At aggregation level 2 that prefix is used to aggregate flow log entries and +is indicated with an asterisk (`*`) at the end of the name. The flows originating from `client-a` and `client-b` are combined into a single flow log. + +Finally, level 3 combines flows between matching pod prefixes that target distinct destination ports, as seen in the last example row below. + +Aggregation is currently performed on a per-node basis, but this behavior may change. Certain other fields are always kept separate +and not aggregated together such as `action` (i.e. denied traffic and allowed traffic will always be logged separately). + +| | | **Src Traffic** | | | **Dst Traffic** | | | **Packet counts** | | +|--------------------------|-----------|----------|---------|----------|----------|---------|----------|------------|-------------| +| **Aggr lvl** | **Flows** | **Name** | **IP** | **Port** | **Name** | **IP** | **Port** | **Pkt in** | **Pkt out** | +| 0 (no aggregation) | 4 | client-a | 1.1.1.1 | 45556 | nginx-1 | 2.2.2.2 | 80 | 1 | 2 | +| | | client-b | 1.1.2.2 | 45666 | nginx-2 | 2.2.3.3 | 80 | 2 | 2 | +| | | client-a | 1.1.1.1 | 65533 | nginx-1 | 2.2.2.2 | 80 | 1 | 3 | +| | | client-c | 1.1.3.3 | 65534 | nginx-2 | 2.2.3.3 | 8080 | 3 | 4 | +| 1 (src port) | 3 | client-a | 1.1.1.1 | - | nginx-1 | 2.2.2.2 | 80 | 2 | 5 | +| | | client-b | 1.1.2.2 | - | nginx-1 | 2.2.2.2 | 80 | 2 | 2 | +| | | client-c | 1.1.3.3 | - | nginx-2 | 2.2.3.3 | 8080 | 3 | 4 | +| 2 (+src/dest pod-prefix) | 2 | client-* | - | - | nginx-* | - | 80 | 4 | 7 | +| | | client-* | - | - | nginx-* | - | 8080 | 3 | 4 | +| 3 (+dest port) | 1 | client-* | - | - | nginx-* | - | - | 7 | 11 | + +## How to + +- [Verify existing aggregation level](#verify-existing-aggregation-level) +- [Change default aggregation level](#change-default-aggregation-level) +- [Troubleshoot logs with aggregation levels](#troubleshoot-logs-with-aggregation-levels) + +### Verify existing aggregation level + +Use the following command: + +```bash +kubectl get felixconfiguration -o yaml +``` + +### Change default aggregation level + +Before [changing the default aggregation level](../../../reference/resources/felixconfig.mdx#aggregationkind), note the following: + +- Although any change in aggregation level affects flow log volume, lowering the aggregation number (especially to `0` for no aggregation) will cause significant impacts to log storage. If you allow more flow logs, ensure that you provision more log storage. +- Verify that the parameters that you want to see in your aggregation level are not already [filtered](filtering.mdx). + +### Troubleshoot logs with aggregation levels + +When you use flow log aggregation, sometimes you may see two Alerts, + +![two-alerts](/img/calico-enterprise/two-alerts.png) + +along with two flow log entries. Note that the entries are identical except for the slight timestamp difference. + +![two-logs](/img/calico-enterprise/two-logs.png) + +The reason you may see two entries is because of the interaction between the aggregation interval, and the time interval to export logs (`flowLogsFlushInterval`). + +In each aggregation interval, connections/connection attempts can be started or completed. However, flow logs do not start/stop when a connection starts/stops. Let’s assume the default export logs “flush” time of 10 seconds. If a connection is started in one flush interval, but terminates in the next, it is recorded across two entries. To get visibility into flow logs to differentiate the entries, go to Service Graph, flow logs tab, and look at these fields: `num_flows`, `num_flows_started`, and `num_flows_completed`. + +The underlying reason for this overlap is a dependency on Linux conntrack, which provides the lifetime of stats that $[prodname] tracks across different protocols (TCP, ICMP, UDP). For example, for UDP and ICMP, $[prodname] waits for a conntrack entry to timeout before it considers a “connection” closed, and this is usually greater than 10 seconds. + +## Additional resources + +- [Archive logs to storage](../archive-storage.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/datatypes.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/datatypes.mdx new file mode 100644 index 0000000000..a422b9d332 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/datatypes.mdx @@ -0,0 +1,184 @@ +--- +description: Reference of key/value fields that Calico Cloud sends to Elasticsearch for flow logs, including endpoints, actions, byte counts, and policy verdicts. +--- + +# Flow log data types + +## Big picture + +$[prodname] sends the following data to Elasticsearch. + +The following table details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html). + +| Name | Datatype | Description | +| --------------------------------- | ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `host` | keyword | Name of the node that collected the flow log entry. | +| `start_time` | date | Start time of log collection in Unix timestamp format. | +| `end_time` | date | End time of log collection in Unix timestamp format. | +| `action` | keyword | - `allow`: $[prodname] accepted the flow.
- `deny`: $[prodname] denied the flow. | +| `bytes_in` | long | Number of incoming bytes since the last export. | +| `bytes_out` | long | Number of outgoing bytes since the last export. | +| `dest_ip` | ip | IP address of the destination pod. A null value indicates aggregation. | +| `dest_name` | keyword | Contains one of the following values:
- Name of the destination pod.
- Name of the pod that was aggregated or the endpoint is not a pod. Check dest_name_aggr for more information, such as the name of the pod if it was aggregated. | +| `dest_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `dest_namespace` | keyword | Namespace of the destination endpoint. A `-` means the endpoint is not namespaced. | +| `dest_port` | long | Destination port. Not applicable for ICMP packets. | +| `dest_service_name` | keyword | Name of the destination service. A `-` means the original destination did not correspond to a known Kubernetes service (e.g. a services ClusterIP). | +| `dest_service_namespace` | keyword | Namespace of the destination service. A `-` means the original destination did not correspond to a known Kubernetes service (e.g. a services ClusterIP). | +| `dest_service_port` | keyword | Port name of the destination service.
A `-` means :
- the original destination did not correspond to a known Kubernetes service (e.g. a services ClusterIP), or
- the destination port is aggregated.
A `*` means there are multiple service port names matching the destination port number. | +| `dest_type` | keyword | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_labels` | array of keywords | Labels applied to the destination pod. A hyphen indicates aggregation. | +| `dest_domains` | array of keywords | Find all the destination domain names for use in a DNS policy by examining `dest_domains`. The field displays information on the top-level domains linked to the destination IP. Applies to flows reported from the source to destinations outside the cluster. If `flowLogsDestDomainsByClient` is disabled, having `dest_domains`: ["A"] doesn't guarantee that the flow corresponds to a connection with domain name A. The destination IP may also be linked to other domain names not yet captured by Calico. | +| `reporter` | keyword | - `src`: flow came from the pod that initiated the connection.
- `dst`: flow came from the pod that received the initial connection.
- `fwd`: flow was forwarded through the node without being the source or destination. | +| `num_flows` | long | Number of flows aggregated into this entry during this export interval. | +| `num_flows_completed` | long | Number of flows that were completed during the export interval. | +| `num_flows_started` | long | Number of flows that were started during the export interval. | +| `num_process_names` | long | Number of unique process names aggregated into this entry during this export interval. | +| `num_process_ids` | long | Number of unique process ids aggregated into this entry during this export interval. | +| `num_process_args` | long | Number of unique process args aggregated into this entry during this export interval. | +| `nat_outgoing_ports` | array of ints | List of [NAT](https://en.wikipedia.org/wiki/Network_address_translation) outgoing ports for the packets that were Source NAT'd in the flow | +| `packets_in` | long | Number of incoming packets since the last export. | +| `packets_out` | long | Number of outgoing packets since the last export. | +| `proto` | keyword | Protocol. | +| `policies` | array of arrays | List of policies that interacted with this flow. See [Format of the policies field](#format-of-the-policies-field). | +| `process_name` | keyword | The name of the process that initiated or received the connection or connection request. This field will have the executable path if flowLogsCollectProcessPath is enabled. A "-" indicates that the process name is not logged. A "\*" indicates that the per flow process limit has exceeded and the process names are now aggregated. | +| `process_id` | keyword | The process ID of the corresponding process (indicated by the `process_name` field) that initiated or received the connection or connection request. A "-" indicates that the process ID is not logged. A "\*" indicates that there are more than one unique process IDs for the corresponding process name. | +| `process_args` | array of strings | The arguments with which the executable was invoked. The size of the list depends on the per flow process args limit. | +| `source_ip` | ip | IP address of the source pod. A null value indicates aggregation. | +| `source_name` | keyword | Contains one of the following values:
- Name of the source pod.
- Name of the pod that was aggregated or the endpoint is not a pod. Check source_name_aggr for more information, such as the name of the pod if it was aggregated. | +| `source_name_aggr` | keyword | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: Endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: the endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `source_namespace` | keyword | Namespace of the source endpoint. A `-` means the endpoint is not namespaced. | +| `source_port` | long | Source port. A null value indicates aggregation. | +| `source_type` | keyword | The type of source endpoint. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload’s own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet’s full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `source_labels` | array of keywords | Labels applied to the source pod. A hyphen indicates aggregation. | +| `original_source_ips` | array of ips | List of external IP addresses collected from requests made to the cluster through an ingress resource. This field is only available if capturing external IP addresses is configured. | +| `num_original_source_ips` | long | Number of unique external IP addresses collected from requests made to the cluster through an ingress resource. This count includes the IP addresses included in the `original_source_ips` field. This field is only available if capturing external IP addresses is configured. | +| `tcp_mean_send_congestion_window` | long | Mean tcp send congestion window size. This field is only available if flowLogsEnableTcpStats is enabled | +| `tcp_min_send_congestion_window` | long | Minimum tcp send congestion window size. This field is only available if flowLogsEnableTcpStats is enabled | +| `tcp_mean_smooth_rtt` | long | Mean smooth RTT in micro seconds. This field is only available if flowLogsEnableTcpStats is enabled | +| `tcp_max_smooth_rtt` | long | Maximum smooth RTT in micro seconds. This field is only available if flowLogsEnableTcpStats is enabled | +| `tcp_mean_min_rtt` | long | Mean MinRTT in micro seconds. This field is only available if flowLogsEnableTcpStats is enabled | +| `tcp_max_min_rtt` | long | Maximum MinRTT in micro seconds. This field is only available if flowLogsEnableTcpStats is enabled | +| `tcp_mean_mss` | long | Mean TCP MSS. This field is only available if flowLogsEnableTcpStats is enabled | +| `tcp_min_mss` | long | Minimum TCP MSS. This field is only available if flowLogsEnableTcpStats is enabled | +| `tcp_total_retransmissions` | long | Total retransmitted packets. This field is only available if flowLogsEnableTcpStats is enabled | +| `tcp_lost_packets` | long | Total lost packets. This field is only available if flowLogsEnableTcpStats is enabled | +| `tcp_unrecovered_to` | long | Total unrecovered timeouts. This field is only available if flowLogsEnableTcpStats is enabled | + +### Format of the policies field + +The `policies` field contains four sub-fields, `all_policies`, `enforced_policies`, `pending_policies`, and `transit_policies`. + +| Name | Datatype | Description | +| --------------------------------- | ----------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `all_policies` | array of keywords | **Deprecated.** This field is a legacy field calculated at flow start time that combines both enforced and staged policies. Because it mixes actual verdicts with simulated ones, it can be misleading and is planned for removal.

**Note:** This field may still appear in logs from older clusters reporting to a newer management cluster. | +| `enforced_policies` | array of keywords | The policies that were actually enforced on the flow. This list is determined by the data plane when the flow starts and remains fixed for the flow's lifetime. It represents the concrete actions taken on the traffic. | +| `pending_policies` | array of keywords | A simulation of what policy evaluation would look like at the time of log generation. It represents a hypothetical restart of the flow where all staged policies are considered active. This field captures types of updates such as: (1) changes to active policies made after the flow started (for example, if a new policy is added that would deny the flow if it restarted), (2) staged policies treated as if they were enforced, and (3) domain-based policies evaluated based on the DNS state at the time of log generation. | +| `transit_policies` | array of keywords | Policies applied to traffic transiting through the node (for example, policies applied to host endpoints with `applyOnForward: true`) or during pre-DNAT processing. This field functions like `enforced_policies` but specifically for traffic being evaluated at the host endpoint.

**Note**: This field is only populated when `flowLogsPolicyScope` is set to `AllPolicies` in the `FelixConfiguration`. | + +Each entry in the list has the following format: + +``` +|||| +``` + +Where, + +* `` numbers the order in which the rules were hit, starting with `0`. + :::tip + Sort the entries of the list by the `` to see the order that rules were hit. The entries are displayed in + random order due to the way they are stored in the datastore. + ::: + +* `` is the name of the policy tier containing the policy, or `__PROFILE__` for a rule derived from a + `Profile` resource (this is the internal datatype used to represent a Kubernetes namespace and its associated + "default allow" rule). +* `` is the name of the policy/profile; its format depends on the type of policy: + + * `.` for $[prodname] `GlobalNetworkPolicy`. + * `/knp.default.` for Kubernetes `NetworkPolicy`. + * `/.` for $[prodname] `NetworkPolicy`. + + Staged policy names are prefixed with "staged:". + +* `` is the action performed by the rule; one of `allow`, `deny`, `pass`. +* `` if non-negative, is the index of the rule that was matched within the policy, starting with 0. + Otherwise, a special value: + + * `-1` means the reporting endpoint was selected by the policy but no rule matched. The traffic hit the default + action for the tier. In this case, the `` is selected arbitrarily from the set of policies within + the tier that apply to the endpoint. + * `-2` means "unknown". The rule index was not recorded. + +### Flow log example, with `no aggregation` + +A flow log with aggregation level 0, `no aggregation`, might look like: + +``` + { + "start_time": 1597166083, + "end_time": 1597166383, + "source_ip": "192.168.47.9", + "source_name": "access-6b687c8dcb-zn5s2", + "source_name_aggr": "access-6b687c8dcb-*", + "source_namespace": "policy-demo", + "source_port": 42106, + "source_type": "wep", + "source_labels": { + "labels": [ + "pod-template-hash=6b687c8dcb", + "app=access" + ] + }, + "dest_ip": "192.168.138.79", + "dest_name": "nginx-86c57db685-h6792", + "dest_name_aggr": "nginx-86c57db685-*", + "dest_namespace": "policy-demo", + "dest_port": 80, + "dest_type": "wep", + "dest_labels": { + "labels": [ + "pod-template-hash=86c57db685", + "app=nginx" + ] + }, + "proto": "tcp", + "action": "allow", + "reporter": "dst", + "policies": { + "all_policies": [ + "0|tier1|ns1/tier1.policy1|pass|3", + "1|tier2|ns2/staged:tier1.policy1|deny|1", + "2|default|policy-demo/default.access-nginx|allow|2" + ], + "enforced_policies": [ + "0|tier1|ns1/tier1.policy1|pass|3", + "1|default|policy-demo/default.access-nginx|allow|2" + ], + "pending_policies": [ + "0|tier1|ns1/tier1.policy1|pass|3", + "1|tier2|ns2/staged:tier1.policy1|deny|1", + ] + }, + "bytes_in": 388, + "bytes_out": 1113, + "num_flows": 1, + "num_flows_started": 1, + "num_flows_completed": 1, + "packets_in": 6, + "packets_out": 5, + "http_requests_allowed_in": 0, + "http_requests_denied_in": 0, + "original_source_ips": null, + "num_original_source_ips": 0, + "host": "bz-n8kf-kadm-node-1", + "@timestamp": 1597166383000 + } +``` + +The log shows an incoming connection reported by the destination node, allowed by a policy on port 80. The **`start_time`** and **`end_time`** +describe the aggregation period (5 min.) During this interval, one flow (**`"num_flow": 1`**) was recorded. At higher aggregation levels, flows from +endpoints performing the same operation and originating from the same Deployment/ReplicaSet are grouped into a single log. In this example, the +common source endpoints that are prefixed with **`access-6b687c8dcb-`**. Parameters like **`source_ip`** may be dropped and set to **`null`**, depending on +the aggregation level. As aggregation levels increase, more flows will be grouped together based on your data. For more details on aggregation +levels, see [configure flow log aggregation](./aggregation.mdx). \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/filtering.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/filtering.mdx new file mode 100644 index 0000000000..15738b32c8 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/filtering.mdx @@ -0,0 +1,109 @@ +--- +description: Filter Calico Cloud flow logs through Fluentd to drop low-significance traffic and reduce managed Elasticsearch volume and cost. +--- + +# Filter flow logs + +## Big picture + +Filter $[prodname] flow logs. + +## Value + +Filter $[prodname] flow logs to suppress logs of low significance, and troubleshoot threats. + +## Concepts + +### Container monitoring tools versus flow logs + +Container monitoring tools are good for monitoring Kubernetes and orchestrated workloads for CPU usage, network usage, and log aggregation. For example, a data monitoring tool can tell if a pod has turned into a bitcoin miner based on it using more than normal CPU. + +$[prodname] flow logs provide continuous records of every single packet sent/received by all pods in your Kubernetes cluster. Note that flow logs do not contain all packet data; only the number of packets/bytes that were sent between specific IP/ports, and when. In the previous monitoring tool example, $[prodname] flow logs could see the packets running to/from the bitcoin mining network. + +$[prodname] flow logs tell you when a pod is compromised, specifically: + +- Where a pod is sending data to +- If the pod is talking to a known command-and-control server +- Other pods that the compromised pod has been talking to (so you can see if they're compromised too) + +### Flow log format + +A flow log contains these space-delimited fields (unless filtered out). + +``` +startTime endTime srcType srcNamespace srcName srcLabels dstType dstNamespace dstName +dstLabels srcIP dstIP proto srcPort dstPort numFlows numFlowsStarted numFlowsCompleted +reporter packetsIn packetsOut bytesIn bytesOut action +``` + +**Example** + +``` +1528842551 1528842851 wep dev rails-81531* - wep dev memcached-38456* - - - 6 - 3000 7 3 4 out 154 61 70111 49404 allow +``` + +- Fields that are not enabled or are aggregated, are noted by `-` +- Aggregated names (such as “pod prefix”), are noted by `*` at the end of the name +- If `srcName` or `dstName` fields contain only a `*`, aggregation was performed using other means (such as specific labels), and no unique prefix was present. + +## How to + +- [Create flow log filters](#create-flow-log-filters) +- [Add filters to ConfigMap file](#add-filters-to-configmap-file) + +### Create flow log filters + +Create your [fluentd filters](https://docs.fluentd.org/filter/grep). + +**Example: filter out a specific namespace** + +This example filters out all flow logs whose source or destination namespace is "dev". Additional namespaces could be filtered by adjusting the regular expression "pattern"s, or by adding additional `exclude` blocks. + +``` + + @type grep + + key source_namespace + pattern dev + + + key dest_namespace + pattern dev + + +``` + +**Example: filter out internet traffic to a specific deployment** + +This example filters inbound internet traffic to the deployment with pods named, `nginx-internet-*`. Note the use of the `and` directive to filter out traffic that is both to the deployment, and from the internet (source `pub`). + +``` + + @type grep + + + key dest_name_aggr + pattern ^nginx-internet + + + key source_name_aggr + pattern pub + + + +``` + +### Add filters to ConfigMap file + +1. Create a `filters` directory with a file called `flow` with your desired filters. If you are also adding [dns filters](../dns/filtering-dns.mdx), add the `dns` file to the directory. + +1. Create the `fluentd-filters` ConfigMap in the `tigera-operator` namespace with the following command. + + ```bash + kubectl create configmap fluentd-filters -n tigera-operator --from-file=filters + ``` + +## Additional resources + +- [Flow log aggregation](aggregation.mdx) +- [Archive logs to storage](../archive-storage.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/hep.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/hep.mdx new file mode 100644 index 0000000000..05505e54de --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/hep.mdx @@ -0,0 +1,33 @@ +--- +description: Turn on host endpoint reporting in Calico Cloud flow logs to gain visibility into traffic at HostEndpoint interfaces on Kubernetes nodes. +--- + +# Enable HostEndpoint reporting in flow logs + +## Big picture + +Enable $[prodname] flow logs to report HostEndpoint information. + +## Value + +Get visibility into the network activity at the HostEndpoint level using $[prodname] flow logs. + +## Before you begin + +**Limitations** + +- HostEndpoint reporting is only supported on Kubernetes nodes. + +## How to + +### Enable HostEndpoint reporting + +To enable reporting HostEndpoint metadata in flow logs, use the following command: + +``` + kubectl patch felixconfiguration default -p '{"spec":{"flowLogsEnableHostEndpoint":true}}' +``` + +## Additional resources + +- [Protect Kubernetes nodes](../../../network-policy/hosts/kubernetes-nodes.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/index.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/index.mdx new file mode 100644 index 0000000000..499213e9b1 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/index.mdx @@ -0,0 +1,11 @@ +--- +description: Configure, filter, and aggregate Calico Cloud flow logs. Add host endpoint, process path, and TCP socket statistics from managed Elasticsearch. +hide_table_of_contents: true +--- + +# Configure flow logs + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/processpath.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/processpath.mdx new file mode 100644 index 0000000000..8d0f9291d0 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/processpath.mdx @@ -0,0 +1,49 @@ +--- +description: Add process executable paths and arguments to Calico Cloud flow logs with eBPF instrumentation for process-level visibility into network activity. +--- + +# Enable process-level information in flow logs + +## Big picture + +Configure $[prodname] to collect process executable path and arguments and add them to flow logs. + +## Value + +Get visibility into the network activity at the process level using $[prodname] flow logs. + +## Concepts + +### eBPF kprobe programs + +eBPF is a Linux kernel technology that allows safe mini-programs to be attached to various hooks inside the kernel. To collect the path and arguments of short-lived processes, this feature uses an eBPF kprobe program. + +### Host's PID namespace + +For long-lived processes, path and arguments are read from `/proc/pid/cmdline`. This requires access to the host's PID namespace. If the access is not available then the process path and arguments will only be captured (by the eBPF kprobes) for newly-created processes. + +## Before you begin + +Ensure that your kernel contains support for eBPF kprobes that $[prodname] uses. The minimum supported +kernel for this is feature is: `v4.4.0`. + +## Privileges + +For full functionality, this feature requires the `$[noderunning]` `DaemonSet` to have access to the host's PID namespace. The Tigera Operator will automatically grant this extra privilege to the daemonset if the feature is enabled in the operator's LogCollector resource, as described below. + +# How to + +### Enable process path and argument collection + +$[prodname] can be configured to enable process path and argument collection on supported Linux kernels +using the command: + +``` + kubectl patch logcollector.operator.tigera.io tigera-secure --type merge -p '{"spec":{"collectProcessPath":"Enabled"}}' +``` + +Enabling/Disabling collectProcessPath causes a rolling update of the `$[noderunning] DaemonSet`. + +The executable path appears in the `process_name` field and `process_args` will have the executable arguments. Executable path +and arguments cannot be collected under certain circumstances, in that `process_name` will have the task name and `process_args` +will be empty. Information about these fields is described in the [Flow log datatype document](datatypes.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/tcpstats.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/tcpstats.mdx new file mode 100644 index 0000000000..330754463b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/flow/tcpstats.mdx @@ -0,0 +1,39 @@ +--- +description: Add TCP socket statistics to Calico Cloud flow logs with eBPF programs that capture round-trip time, retransmits, and other per-socket metrics. +--- + +# Enabling TCP socket stats in flow logs + +## Big picture + +Configure $[prodname] to collect additional TCP socket statistics. While this feature is available in both iptables and eBPF data plane modes, it uses eBPF to collect the statistics. Therefore it requires a recent Linux kernel (at least v5.10.0/v4.18.0-305 for RHEL). + +## Value + +Get visibility into the network activity at the socket level using $[prodname] flow logs. + +## Concepts + +### eBPF TC programs + +eBPF is a Linux kernel technology that allows safe mini-programs to be attached to various hooks inside the kernel. This feature leverages eBPF to look up the TCP socket associated with packets flowing through an interface and sends them to userspace for addition to flow logs. + +## Before you begin + +Ensure that your kernel contains support for eBPF that $[prodname] uses. The minimum supported +kernel for tcp socket stats is: `v5.10.0`. For distros based on RHEL, the minimum kernel version is `v4.18.0-305`. + +# How to + +### Enable tcp stats collection + +$[prodname] can be configured to enable tcp socket stats collection on supported Linux kernels +using the command: + +``` + kubectl patch felixconfiguration default -p '{"spec":{"flowLogsCollectTcpStats":true}}' +``` + +The additional fields collected are `tcp_mean_send_congestion_window`, `tcp_min_send_congestion_window`, `tcp_mean_smooth_rtt`, `tcp_max_smooth_rtt`, +`tcp_mean_min_rtt`, `tcp_max_min_rtt`, `tcp_mean_mss`, `tcp_min_mss`, `tcp_total_retransmissions`, `tcp_lost_packets`, `tcp_unrecovered_to`. +Information about these fields is described in the [Flow log datatype document](datatypes.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/index.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/index.mdx new file mode 100644 index 0000000000..23350448f2 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/index.mdx @@ -0,0 +1,11 @@ +--- +description: Configure managed Elasticsearch logs for Calico Cloud to surface flow, DNS, audit, and L7 data from connected clusters in the web console. +hide_table_of_contents: true +--- + +# Manage Calico Cloud logs + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/configure.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/configure.mdx new file mode 100644 index 0000000000..85f19a5372 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/configure.mdx @@ -0,0 +1,167 @@ +--- +description: Deploy Envoy and aggregate Calico Cloud L7 logs to monitor HTTP traffic patterns between application workloads on connected clusters. +--- + +# Configure L7 logs + +:::warning[deprecation and removal notice] + +L7 logging with Envoy is deprecated and will be removed in an upcoming release. +$[prodname] is moving to L7 observability based on [Istio Ambient Mode](../../../compliance/istio/about-istio-ambient.mdx). + +::: + +## Big picture + +Deploy Envoy and use $[prodname] L7 logs to monitor application activity. + +## Value + +Just like L3/4 $[prodname] logs, platform operators and +development teams want visibility into L7 logs to see how applications are interacting with each +other. $[prodname] flow logs only display which workloads are communicating +with each other, not the specific request details. $[prodname] provides visibility into L7 traffic without the need for a service mesh. + +L7 logs are also key for detecting anomalous behaviors like attempts to +access applications, restricted URLs, and scans for particular URLs. + +## Concepts + +### About L7 logs + +L7 logs capture application interactions from HTTP header data in requests. Data shows what is actually sent in communications between specific pods, providing more specificity than flow logs. (Flow logs capture data only from connections for workload interactions). + +$[prodname] collects L7 logs by sending the selected traffic through an Envoy proxy. + +L7 logs are visible in the web console, service graph, in the HTTP tab. + +## Before you begin + +**Not supported** +- GKE + +### Limitations + +* L7 log collection is not compatible with a service mesh such as Istio. +* L7 log collection is not supported for host-networked client pods. +* L7 log collection is supported only on Kubernetes 1.29 and later. + +:::important + +When you enable L7 log collection for a deployment, all of that deployment's pods will restart. +Selecting and deselecting traffic for L7 log collection may result in disruptions for active connections. + +::: + +:::note + +L7 logs require a minimum of 1 additional GB of log storage per node, per one-day retention period. + +::: + +## Configure Felix for log data collection + +1. Configure L7 log aggregation, retention, and reporting. + + For help, see [Felix Configuration documentation](../../../reference/component-resources/node/felix/configuration.mdx). + +## Configure L7 logs + +In this step, you will configure L7 logs, select logs for collection, and test the configuration. + +### Configure the ApplicationLayer resource for L7 logs + +1. Create or update the [ApplicationLayer](../../../reference/installation/api.mdx#applicationlayer) resource named, `tigera-secure`. + + Example: + + ```bash + kubectl apply -f - <` flag, it is going to be installed in the `default` namespace. if you wish to install in a different namespace, specify the `-n ` flag on all other commands in this tutorial from this point on. + +1. Patch the `frontend` application deployment to start collecting L7 logs for this application + + ```bash + kubectl patch deployment frontend -p '{"spec":{"template":{"metadata":{"labels":{"applicationlayer.projectcalico.org/sidecar":"true"},"annotations":{"applicationlayer.projectcalico.org/logging":"Enabled"}}}}}' + ``` + + The above command applies the following patch: + 1. ensures pods in your `frontend` deployment pod template spec has the label `applicationlayer.projectcalico.org/sidecar: true` + 1. ensures pods in your `frontend` deployment pod template spec has the following annotations: + - `applicationlayer.projectcalico.org/logging: Enabled` + + Deployment pods will then be expected to restart. Monitor its progress with: + + ```bash + kubectl rollout status deployment frontend + ``` + + [Google Microservices Demo](https://github.com/GoogleCloudPlatform/microservices-demo) includes a load generator. L7 logs should now start displaying log entries that can be viewed via [various UIs](#view-l7-logs-in-the-web-console) + + Alternatively, it is possible to test your configuration [manually](#test-your-configuration). + +1. To disable L7 log collection, reverse the above patch with the following command. + + ```bash + kubectl patch deployment frontend --type='json' -p '[{"op":"remove","path":"/spec/template/metadata/labels/applicationlayer.projectcalico.org~1sidecar"},{"op":"remove","path":"/spec/template/metadata/annotations/applicationlayer.projectcalico.org~1logging"}]' + ``` + +### Test your configuration + +1. Identify the path to access your cluster. Where `` can be: + + - Public address of your cluster/service + or + - Cluster IP of your application's service (if testing within the cluster) + +1. `curl` your service with a command similar to the following. You will see `Server` header as `envoy`. + + ```bash + curl --head :/ + ``` + +## View L7 logs in the web console + +### Service Graph + +To view L7 logs in Service Graph: + +1. In the web console left navbar, click **Service Graph**. +1. In the bottom pane you will see L7 logs in the HTTP tab. + + ![l7-logs](/img/calico-enterprise/l7-logs.png) + diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/datatypes.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/datatypes.mdx new file mode 100644 index 0000000000..8db8eb8309 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/datatypes.mdx @@ -0,0 +1,47 @@ +--- +description: Reference of key/value fields that Calico Cloud sends to Elasticsearch for L7 logs, with the collector that populates each field. +--- + +# L7 log data types + +$[prodname] supports multiple L7 collectors that write into the same Elasticsearch index: + +- **Envoy** — the legacy sidecar/proxy-based collector. See [Enable Envoy collector](configure.mdx). +- **eBPF** — kernel TCP-layer probes capturing HTTP request/response and TLS handshake metadata. See [Enable eBPF collector](enable-ebpf-collector.mdx). +- **Waypoint** — Istio Ambient Mode HTTP logging via the Waypoint Proxy. See [Enable Waypoint collector](enable-waypoint-collector.mdx). + +The Envoy and Waypoint collectors are both built on Envoy access logs, and the table below refers to them collectively as the _Envoy-based_ collectors. They share the same `collector_type` value (`envoy-access-log`) and are told apart by `collector_name`. + +The table below details the key/value pairs in the JSON blob, including their [Elasticsearch datatype](https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) and which collector(s) populate each field. "All" means every collector populates the field. Descriptions prefixed with _HTTP only_ or _TLS only_ mark fields that are populated for events of only that protocol family. + +| Name | Datatype | Populated by | Description | +| ------------------------ | -------- | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `host` | keyword | All | Name of the node that collected the L7 log entry. | +| `start_time` | date | All | Start time of log collection in Unix timestamp format. | +| `end_time` | date | All | End time of log collection in Unix timestamp format. | +| `bytes_in` | long | All | Number of incoming bytes since the last export. | +| `bytes_out` | long | All | Number of outgoing bytes since the last export. | +| `duration_mean` | long | All | Mean duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `duration_max` | long | All | Max duration time of all the requests that match this combination of L7 data in nanoseconds. | +| `count` | long | All | Number of requests that match this combination of L7 data. | +| `src_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the source pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `src_namespace` | keyword | All | Namespace of the source endpoint. | +| `src_type` | keyword | All | Source endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, the one with the longest-prefix match is chosen. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_name_aggr` | keyword | All | Contains one of the following values:
- Aggregated name of the destination pod.
- `pvt`: endpoint is not a pod. Its IP address belongs to a private subnet.
- `pub`: endpoint is not a pod. Its IP address does not belong to a private subnet. It is probably an endpoint on the public internet. | +| `dest_namespace` | keyword | All | Namespace of the destination endpoint. | +| `dest_type` | keyword | All | Destination endpoint type. Possible values:
- `wep`: A workload endpoint, a pod in Kubernetes.
- `ns`: A network set. If multiple match, priority is given to NetworkSets in the workload's own namespace, then to GlobalNetworkSets, and then to NetworkSets in other namespaces. For ties between matching network sets within each category, CIDR matches outrank domain matches and longest-prefix wins between competing CIDR matches. Remaining ties are resolved alphabetically by the NetworkSet's full identity (using namespace/name or just name).
- `net`: A Network. The IP address did not fall into a known endpoint type. | +| `dest_service_name` | keyword | All | Name of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_namespace` | keyword | All | Namespace of the destination service. This may be empty if the request was not made against a service. | +| `dest_service_port` | long | All | Destination service port. | +| `url` | keyword | All | _HTTP only._ URL that the request was made against. | +| `response_code` | keyword | All | _HTTP only._ Response code returned by the request. | +| `method` | keyword | All | _HTTP only._ HTTP method for the request. | +| `user_agent` | keyword | All | _HTTP only._ User agent of the request. | +| `type` | keyword | All | Type of request made. Possible values include `tcp`, `tls`, and `html/`. | +| `protocol` | keyword | All | Wire-protocol family of the captured event. Possible values: `http`, `tls`. | +| `protocol_version` | keyword | All | Wire-protocol version. For HTTP events, possible values include `1.0`, `1.1`, and `2`. | +| `collector_type` | keyword | All | Technology family of the collector that produced the event. Possible values: `ebpf` for the eBPF collector, and `envoy-access-log` for the Envoy-based collectors (Envoy and Waypoint). | +| `collector_name` | keyword | All | Specific collector that produced the event within the family. Possible values: `ebpf-tcp` (eBPF kernel TCP-layer collector), `daemonset` (Envoy collector), and `waypoint` (Waypoint collector). | +| `tls_sni` | keyword | eBPF | _TLS only._ TLS Server Name Indication sent by the client in the ClientHello. | +| `tls_version` | keyword | eBPF | _TLS only._ Negotiated TLS protocol version (for example, `TLS 1.2`, `TLS 1.3`). | +| `tls_cipher_suite` | keyword | eBPF | _TLS only._ Negotiated TLS cipher suite (for example, `TLS_AES_256_GCM_SHA384`). | diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/enable-ebpf-collector.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/enable-ebpf-collector.mdx new file mode 100644 index 0000000000..07dbde6c27 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/enable-ebpf-collector.mdx @@ -0,0 +1,40 @@ +--- +description: Enable the kernel TCP-layer eBPF collector to capture HTTP and TLS L7 logs without a proxy or sidecar. +--- + +# Collect L7 logs using the eBPF collector + +Enable the kernel TCP-layer eBPF collector to capture HTTP request/response and TLS handshake metadata without deploying a proxy or injecting sidecars. You do not need the eBPF data plane to use it — the collector works with any $[prodname] data plane (iptables, nftables, or eBPF). For background on what the collector does and how it compares to other options, see [L7 logs overview](overview.mdx). + +## Limitations + +* HTTP parsing covers HTTP/1.0 and HTTP/1.1 only, because their requests and responses travel as plain text on the wire. HTTP/2 and HTTP/3 are not captured as HTTP, and encrypted HTTP traffic surfaces only as TLS metadata. +* HTTP capture tracks only the most recent request per socket; when a single connection carries multiple back-to-back requests, some may be missed. +* TLS capture is limited to handshake metadata (SNI, version, cipher suite). The collector does not decrypt application traffic. + +## Prerequisites + +* Linux kernel 5.17 or later on every node. +* Additional log storage for the L7 logs. + L7 logs require a minimum of 1 additional GB of log storage per node, per one-day retention period. + +## Enable the collector + +1. Enable the eBPF L7 collector by setting the [L7ObservabilityEnabled](../../../reference/component-resources/node/felix/configuration.mdx#L7ObservabilityEnabled) field to `true` on the default `FelixConfiguration`. This setting is independent of the `bpfEnabled` field, so enabling the collector does not switch your data plane: + + ```bash + kubectl patch felixconfiguration default --type=merge -p '{"spec":{"l7ObservabilityEnabled":true}}' + ``` + +2. Verify that L7 logs are being collected by looking for output in the default log file location (`/var/log/calico/l7logs/l7.log`) on each node. Logs are flushed on the interval set by the [L7LogsFlushInterval](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFlushInterval) field, so allow one interval to pass before you expect output: + + ```bash + tail -f /var/log/calico/l7logs/l7.log + ``` + + The log directory is controlled by the [L7LogsFileDirectory](../../../reference/component-resources/node/felix/configuration.mdx#L7LogsFileDirectory) field on `FelixConfiguration`. + +## Additional information + +* [L7 log data types](datatypes.mdx) +* [Log aggregation](../aggregation.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/enable-waypoint-collector.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/enable-waypoint-collector.mdx new file mode 100644 index 0000000000..5a6cb03d24 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/enable-waypoint-collector.mdx @@ -0,0 +1,90 @@ +--- +description: Enable the Istio Ambient Mode Waypoint-based L7 collector. +--- + +# Collect L7 logs using the Waypoint collector + +Enable the Istio Ambient Mode Waypoint-based L7 collector to capture HTTP/1, HTTP/2, and gRPC traffic. + +## Prerequisites + +- A workload running in Calico's bundled [Istio Service Mesh](../../../compliance/istio/deploy-istio-ambient.mdx). + +## Enable the collector + +Once a workload is set up to be in the service mesh, L7 collection is enabled +by adding a [waypoint proxy](https://istio.io/latest/docs/ambient/usage/waypoint/), +and labeling the namespace to use it. + +This will create an Istio Waypoint proxy that includes a sidecar container to +capture the L7 logs. + +```bash +cat << EOF | kubectl apply -n -f - +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: + labels: + istio.io/waypoint-for: all +spec: + gatewayClassName: istio-waypoint + listeners: + - name: mesh + port: 15008 + protocol: HBONE +EOF +``` + +```bash +kubectl label namespace istio.io/use-waypoint= +``` + +Replace the following: + +- ``: The namespace your workload is in. +- ``: What you want your waypoint proxy gateway to be called. + +## Verify the collector is running + +```bash +kubectl get pods -n -l gateway.networking.k8s.io/gateway-name= \ + -o jsonpath='{range.items[*]}{.metadata.name}{"\t"}{.status.phase}{"\t"}{.spec.containers[*].name}{"\n"}{end}' +``` + +You should see both the `istio-proxy` and `l7-collector` containers listed. + +## Remove the Waypoint collector + +```bash +kubectl label namespace istio.io/use-waypoint- +``` + +## Disable the Waypoint collector + +If you are running waypoint proxies but you don't want sidecars added, the +Waypoint collector sidecar can be disabled: + +```bash +kubectl patch istio default --type=merge -p '{"spec":{"waypointLogging":"Disabled"}}' +``` + +Note that this applies to all waypoint proxies in the cluster. + +## View L7 logs + +Waypoint collector logs are viewed in the [UI for L7 logs](configure.mdx#view-l7-logs-in-the-web-console) +along with the other types of L7 logs. + +They will be labeled with the `collector_type`: `envoy-access-log` and the +`collector_name`: `waypoint`. + +:::note + +L7 logs take several minutes to appear in the web console after the Waypoint collector is enabled. + +::: + +## Additional information + +- [L7 log data types](datatypes.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/index.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/index.mdx new file mode 100644 index 0000000000..d3b80a9a85 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/index.mdx @@ -0,0 +1,11 @@ +--- +description: Configure L7 application traffic logs for Calico Cloud. Deploy Envoy, set aggregation, and review the Elasticsearch L7 log schema for connected clusters. +hide_table_of_contents: true +--- + +# L7 logs + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/overview.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/overview.mdx new file mode 100644 index 0000000000..cf08d1bea6 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/l7/overview.mdx @@ -0,0 +1,45 @@ +--- +description: Overview of Calico Cloud L7 logs, the available collectors, and how to choose between them. +--- + +# L7 logs + +$[prodname] L7 logs capture HTTP request/response and TLS handshake metadata for traffic between workloads in your cluster. They show what was actually sent (method, URL, response code, SNI, TLS version) between specific pods, going beyond the L3/4 flow log view of who-talked-to-whom. + +## Value + +Platform operators and development teams use L7 logs to: + +- See how applications interact, request by request. +- Spot anomalous behavior — attempts to access restricted URLs, unexpected user agents, scans for particular paths. +- Inspect TLS sessions by SNI, negotiated version, and cipher suite. + +L7 logs complement L3/4 flow logs, which expose only the source/destination pair without request-level detail. + +## Collectors + +A _collector_ is the component that observes traffic in your cluster and emits L7 log entries. $[prodname] supports multiple collectors. They all feed the same aggregation and reporting pipeline; each entry on the log file is tagged with `collector_type`. + +- **eBPF collector** — kernel TCP-layer probes that capture HTTP and TLS metadata with no data path proxy or sidecar. Enable once at the cluster level. Works with any data plane — you do not need the eBPF data plane to use it. See [Enable eBPF collector](enable-ebpf-collector.mdx). +- **Waypoint collector** — Waypoint proxy based Envoy sidecar. See [Enable Waypoint collector](enable-waypoint-collector.mdx). +- **Envoy collector** _(legacy, deprecated)_ — sidecar Envoy injected per workload. Being phased out. See [Enable Envoy collector](configure.mdx). + +### Why more than one collector? + +Each collector lives at a different point in the request path and exposes different trade-offs. Pick by where you need coverage and what infrastructure you're willing to run: + +| | eBPF | Envoy (legacy) | Waypoint | +| -------------------------- | --------------------------------- | --------------------------------- | -------------------------------- | +| Additional infrastructure | None — runs inside Felix | Sidecar Envoy per workload | Sidecar Envoy per Waypoint proxy | +| Protocol coverage | HTTP/1.x, TLS handshake | HTTP | HTTP/1, HTTP/2, gRPC | +| Opt-in granularity | Cluster-wide (FelixConfiguration) | Per workload (label + annotation) | Per namespace (label) | +| Service-mesh compatibility | Any data plane | Not compatible with Istio | Istio Ambient Mode only | + +Multiple collectors can coexist in the same cluster, with each L7 log entry tagged by `collector_type` so consumers can filter or join sources as needed. + +## Next steps + +- [Enable eBPF collector](enable-ebpf-collector.mdx) +- [Enable Waypoint collector](enable-waypoint-collector.mdx) +- [L7 log data types](datatypes.mdx) +- [Log aggregation](../aggregation.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/observability/elastic/overview.mdx b/calico-cloud_versioned_docs/version-24-1/observability/elastic/overview.mdx new file mode 100644 index 0000000000..d5e9d913f6 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/elastic/overview.mdx @@ -0,0 +1,87 @@ +--- +description: Calico Cloud uses managed Elasticsearch for flow, DNS, audit, BGP, and L7 logs with workload context, RBAC, and archival to external SIEMs. +--- + +# Overview + +## Big picture + +Use $[prodname] log data for visibility and troubleshooting Kubernetes clusters. + +## Value + +Workloads and policies are highly dynamic. To troubleshoot Kubernetes clusters, you need logs with workload identity and context. $[prodname] deploys an Elasticsearch cluster during installation with these features: + +- Logs with workload context +- Centralized log collection for multiple clusters for $[prodname] multi-cluster-management +- View Elasticsearch logs in the $[prodname] web console (Flow Visualizer), and the [Elasticsearch API](https://www.elastic.co/guide/en/elasticsearch/reference/current/search.html) +- Standard Kubernetes RBAC for granular access control to logs +- Collect/archive logs or subset of logs +- Log aggregation for high-volume logs +- Configure data retention settings to manage cluster disk space +- Integration with third-party tools like Amazon S3, Syslog, Splunk + +## Concepts + +### Logs types + +Elasticsearch logs provide the visibility and troubleshooting backend for $[prodname]. + +| Log type | Description | Log source | RBAC | Index | +| -------- | ---------------------------------------------------------------------------------------------- | ------------------------------------------- | ------------ | ----------------------------- | +| flow | Network flows for workloads: source and destination namespaces, pods, labels, and policies | $[prodname] cnx-node (Felix) | `flows` | `tigera_secure_ee_flows` | +| audit | Audit logs for $[prodname] resources | $[prodname] apiserver | `audit_ee` | `tigera_secure_ee_audit_ee` | +| | Audit logs for Kubernetes resources | Kubernetes apiserver | `audit_kube` | `tigera_secure_ee_audit_kube` | +| | | Both audit logs above | `audit*` | `tigera_secure_ee_audit*` | +| bgp | $[prodname] networking BGP peering and route propagation | $[prodname] cnx-node (BIRD) | `ee_bgp` | `tigera_secure_ee_bgp.*` | +| dns | DNS lookups and responses from $[prodname] domain-based policy | $[prodname] cnx-node (Felix) | `ee_dns` | `tigera_secure_ee_dns` | +| ids | $[prodname] intrusion detection events: suspicious IPs, suspicious domains, and global alerts | $[prodname] intrusion-detection-controller | `ee_events` | `tigera_secure_ee_events` | + +:::note + +Because of their high-volume, flow and dns logs support aggregation. + +::: + +### Default log configuration and security + +$[prodname] automatically installs fluentd on all nodes and collects flow, audit, and DNS logs. You can configure additional destinations like Amazon S3, Syslog, Splunk. + +$[prodname] enables user authentication in Elasticsearch, and secures access to Elasticsearch using network policy. + +### RBAC and log access + +You control user access to logs using the standard Kubernetes RBAC cluster role and cluster role binding. For example: + +```yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: bob-es-access +subjects: + - kind: User + name: bob + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: audit-ee-only + apiGroup: rbac.authorization.k8s.io +``` + +You configure Elasticsearch log access per cluster using RBAC and the Kubernetes API group, `lma.tigera.io`. For example: + +```yaml +apiGroups: ['lma.tigera.io'] +resources: ['app-cluster'] +resourceNames: ['flows', 'dns'] +verbs: ['get'] +``` + +## Additional resources + +- [Configure flow log aggregation](flow/aggregation.mdx) +- [Audit logs](audit-overview.mdx) +- [BGP logs](bgp.mdx) +- [DNS logs](dns/dns-logs.mdx) +- [Archive logs](archive-storage.mdx) +- [Log collection options](../../reference/installation/api.mdx#logcollectorspec) diff --git a/calico-cloud_versioned_docs/version-24-1/observability/index.mdx b/calico-cloud_versioned_docs/version-24-1/observability/index.mdx new file mode 100644 index 0000000000..0630721567 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/index.mdx @@ -0,0 +1,54 @@ +--- +description: Observe and troubleshoot connected clusters with Calico Cloud web console dashboards, Service Graph, packet capture, and managed Elasticsearch logs. +hide_table_of_contents: true +--- + +import { DocCardLink, DocCardLinkLayout } from '/src/___new___/components'; + +# Observability and troubleshooting + +See what's going on in your cluster with network observability tools and detailed logging. + +## Getting started + + + + + + + +## Getting started with logs + + + + + + + + + +## Flow logs + + + + + + + + + + + +## DNS logs + + + + + + +## L7 logs + + + + + diff --git a/calico-cloud_versioned_docs/version-24-1/observability/iptables.mdx b/calico-cloud_versioned_docs/version-24-1/observability/iptables.mdx new file mode 100644 index 0000000000..7d4c5da111 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/iptables.mdx @@ -0,0 +1,61 @@ +--- +description: Reference explaining how Calico Cloud policy audit mode and the Log rule action influence iptables log volume on connected cluster nodes. +--- + +# iptables logs + +## About iptables logs + +iptables logs are produced by [policy audit mode](#policy-audit-mode) or by using the `Log` action in either +[Network Policy](../reference/resources/networkpolicy.mdx) or [Global Network Policy](../reference/resources/globalnetworkpolicy.mdx). +These logs are written to syslog (specifically the `/dev/log` socket) on the nodes where the events are generated. +Collection, rotation and other management of these logs is provided by your syslog agent, for example, journald or rsyslogd. + +## Policy audit mode + +$[prodname] adds a Felix option `DropActionOverride` that configures how the +`deny` `action` in a [Rule](../reference/resources/networkpolicy.mdx#rule) is interpreted. +It can add logs for denied packets, or even allow the traffic through. + +See the +[Felix configuration reference](../reference/component-resources/node/felix/configuration.mdx) for +information on how to configure this option. + +`DropActionOverride` controls what happens to each packet that is denied by +the current $[prodname] policy, i.e., by the ordered combination of all the +configured policies and profiles that apply to that packet. It may be +set to one of the following values: + +- `Drop` +- `Accept` +- `LogAndDrop` +- `LogAndAccept` + +Normally the `Drop` or `LogAndDrop` value should be used, as dropping a +packet is the obvious implication of that packet being denied. However when +experimenting, or debugging a scenario that is not behaving as you expect, the +"Accept" and "LogAndAccept" values can be useful: then the packet will be +still be allowed through. + +When one of the `LogAnd*` values is set, each denied packet is logged in +syslog, with an entry like this: + +``` +May 18 18:42:44 ubuntu kernel: [ 1156.246182] calico-drop: IN=tunl0 OUT=cali76be879f658 MAC= SRC=192.168.128.30 DST=192.168.157.26 LEN=60 TOS=0x00 PREC=0x00 TTL=62 ID=56743 DF PROTO=TCP SPT=56248 DPT=80 WINDOW=29200 RES=0x00 SYN URGP=0 MARK=0xa000000 +``` + +Note that [Denied Packet Metrics](../operations/monitor/metrics/index.mdx) are independent of the `DropActionOverride` +setting. Specifically, if packets that would normally be denied are being +allowed through by a setting of `Accept` or `LogAndAccept`, those packets +still contribute to the denied packet metrics as normal. + +For example, to set a `DropActionOverride` for `myhost` to log then drop denied packets: + +Edit the FelixConfiguration object for the `myhost` Node. + +```bash +kubectl patch felixconfiguration.p node.myhost --type='merge' -p \ + '{"spec":{"dropActionOverride":"LogAndDrop"}}' +``` + +For a global setting, modify the `default` FelixConfiguration resource. diff --git a/calico-cloud_versioned_docs/version-24-1/observability/kube-audit.mdx b/calico-cloud_versioned_docs/version-24-1/observability/kube-audit.mdx new file mode 100644 index 0000000000..8129ca62c4 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/kube-audit.mdx @@ -0,0 +1,281 @@ +--- +description: Turn on Kubernetes API audit logging for Calico Cloud connected clusters so security teams can review changes to pods, namespaces, and network policies. +--- + +# Kubernetes audit logs + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## Big picture + +Enable Kubernetes audit logs so security teams and auditors can see all the changes to Kubernetes resources over time. + +## Concepts + +### About Kubernetes audit logs + +**Kubernetes resources** are used in $[prodname] audit-related features, but they are not enabled by default. You must enable Kubernetes resources through the Kubernetes API server. If you miss this step, audit trails will not provide a complete view to your security team. + +You must enable the following Kubernetes resources for each cluster: + +- Pod +- Namespace +- Service account +- Network policy +- Endpoints + +### Audit logs in the web console + +Like $[prodname] audit logs, Kubernetes audit logs are displayed in the web console in the Timeline dashboard. + +## Before you begin + +**Unsupported** + +- AKS +- GKE +- OpenShift +- TKG + +## How to + +Enable Kubernetes audit logs in the Kubernetes API server: + + + + +### Enable audit logs for Kubernetes resources + +At a minimum, enable audit logs for these resources that are involved in network policy: + +- Pod +- Namespace +- ServiceAccount +- NetworkPolicy (Kubernetes/OpenShift) +- Endpoints + +**Sample policy** + +The following sample policy audits changes to Kubernetes Pod, Namespace, ServiceAccount, Endpoints and NetworkPolicy resources. To add other audit logs for resources beyond network policy, see the [Kubernetes docs](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/), or review this function for inspiration (which generates the GKE audit policy). + +```yaml +apiVersion: audit.k8s.io/v1beta1 +kind: Policy +omitStages: + - RequestReceived +rules: + - level: RequestResponse + verbs: + - create + - patch + - update + - delete + resources: + - group: networking.k8s.io + resources: ['networkpolicies'] + - group: extensions + resources: ['networkpolicies'] + - group: '' + resources: ['pods', 'namespaces', 'serviceaccounts', 'endpoints'] +``` + +### Enable Kubernetes audit logs for $[prodname] + +The following updates require a restart to the Kubernetes API Server. + +To enable Kubernetes resource audit logs to be read by $[prodname] in fluentd, follow these steps. + +On the Kubernetes API Server, update these flags. + +- `--audit-log-path=/var/log/calico/audit/kube-audit.log` +- `--audit-policy-file=` + For help with flags, see kube-apiserver flags. For help with audit logging, see Kubernetes audit logging documentation. + +Distribute the audit policy file to all control plane nodes, ensuring that it is available to the Kubernetes API server (e.g. by volume mounting it into the pods). + +Restart the Kubernetes API server. The restart command depends on how you installed Kubernetes. + + + + +### Enable audit logs for Kubernetes resources + +At a minimum, enable audit logs for these resources that are involved in network policy: + +- Pod +- Namespace +- ServiceAccount +- NetworkPolicy (Kubernetes/OpenShift) +- Endpoints + +**Sample policy** + +The following sample policy audits changes to Kubernetes Pod, Namespace, ServiceAccount, Endpoints and NetworkPolicy resources. To add other audit logs for resources beyond network policy, see the [Kubernetes docs](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/), or review this function for inspiration (which generates the GKE audit policy). + +```yaml +apiVersion: audit.k8s.io/v1beta1 +kind: Policy +omitStages: + - RequestReceived +rules: + - level: RequestResponse + verbs: + - create + - patch + - update + - delete + resources: + - group: networking.k8s.io + resources: ['networkpolicies'] + - group: extensions + resources: ['networkpolicies'] + - group: '' + resources: ['pods', 'namespaces', 'serviceaccounts', 'endpoints'] +``` + +### Enable audit logs in EKS + +Amazon EKS writes Kubernetes audit logs to [Amazon CloudWatch logs](https://aws.amazon.com/cloudwatch/). + +1. In the EKS management console, access your EKS cluster. +1. Under **Logging**, click **Update**. +1. Enable the **Audit** option, and click **Update**. + Audit Log +1. Wait for the update to complete. + The blue progress bar at the top of the page displays the message, “Cluster config update in progress.” +1. Under **Logging**, **CloudWatch**, make a note of the URL value for a later step, then click the link. + CloudWatch Logging +1. In the **Log Streams** list, make note of the common prefix (for example, kube-apiserver-audit) for a later step. + Log Streams +1. Make note of the region where the cluster is hosted (for example, `us-west-2`) for a later step. + +### Create a restricted AWS user for audit log access + +1. Go to the **AWS IAM console** and add a user. +1. On the **Add user** page, make these changes: + + a. Select **Access type**, **Programmatic access**. + + Programmatic access + + b. In the Set permissions section, select the policy, `CloudWatchLogsReadOnlyAccess` to set read only permissions. + + CloudWatch URL + +1. Optional: In the **Add tags** section, add a tag for the user based on your cluster information. +1. Click **Submit** to create a restricted user. + +### Update $[prodname] log collector with EKS values + +1. Update the `tigera-secure` LogCollector resource with values from the EKS configuration. + + where: + + - `additionalSources`: Section where EKS CloudWatch logs are specified. + - `eksCloudwatchLog`: Configuration section containing EKS CloudWatch logs. + - `fetchInterval`: Interval in seconds for $[prodname] to get logs from CloudWatch. Default: 60 seconds, this fetches 1MB every 60 seconds, adjust it based number on CRUD operations performed on cluster resource. + - `groupName`: Name of the `Log Group` (value from "Enable audit logs in EKS") + - `region`: AWS region where EKS cluster is hosted (value from "Enable audit logs in EKS") + - `streamPrefix`: Prefix of `Log Stream` (value from "Enable audit logs in EKS") + + **Example** + + ```yaml + apiVersion: operator.tigera.io/v1 + kind: LogCollector + metadata: + name: tigera-secure + spec: + additionalSources: + eksCloudwatchLog: + fetchInterval: 60 + groupName: /aws/eks/mitch-eks-kube-audit-log-forwarder/cluster + region: us-west-2 + streamPrefix: kube-apiserver-audit- + status: + state: Ready + ``` + +### Configure authentication between $[prodname] and CloudWatch logs + +In this step, you add AWS authentication information to enable $[prodname] to get logs from the EKS CloudWatch instance. + +Add a Secret with the name, `tigera-eks-log-forwarder-secret` in the namespace, `tigera-operator`, and the AWS [Security Credentials](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html) in the data section. + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: tigera-eks-log-forwarder-secret + namespace: tigera-operator +type: Opaque +data: + aws-id: $(echo -n | base64 -w0) + aws-key: $(echo -n | base64 -w0) +``` + + + + +### Enable audit logs for Kubernetes resources + +At a minimum, enable audit logs for these resources that are involved in network policy: + +- Pod +- Namespace +- ServiceAccount +- NetworkPolicy (Kubernetes/OpenShift) +- Endpoints + +**Sample policy** + +The following sample policy audits changes to Kubernetes Pod, Namespace, ServiceAccount, Endpoints and NetworkPolicy resources. To add other audit logs for resources beyond network policy, see the [Kubernetes docs](https://kubernetes.io/docs/tasks/debug/debug-cluster/audit/), or review this function for inspiration (which generates the GKE audit policy). + +```yaml +apiVersion: audit.k8s.io/v1beta1 +kind: Policy +omitStages: + - RequestReceived +rules: + - level: RequestResponse + verbs: + - create + - patch + - update + - delete + resources: + - group: networking.k8s.io + resources: ['networkpolicies'] + - group: extensions + resources: ['networkpolicies'] + - group: '' + resources: ['pods', 'namespaces', 'serviceaccounts', 'endpoints'] +``` + +Follow these instructions to enable audit logs for [AWS using kOps](https://kops.sigs.k8s.io/cluster_spec/#audit-logging). + +Note that `auditLogPath` should be `/var/log/calico/audit/kube-audit.log`. + + + diff --git a/calico-cloud_versioned_docs/version-24-1/observability/packetcapture.mdx b/calico-cloud_versioned_docs/version-24-1/observability/packetcapture.mdx new file mode 100644 index 0000000000..e99031de8e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/packetcapture.mdx @@ -0,0 +1,346 @@ +--- +description: Capture live pod traffic in Calico Cloud connected clusters from Service Graph or the CLI and export pcap files to Wireshark for analysis. +--- + +# Packet capture + +## Big picture + +Capture live traffic inside a Kubernetes cluster, and export to visualization tools like Wireshark for troubleshooting and debugging applications. + +## Value + +$[prodname] packet capture is implemented in a Kubernetes-native way so you can troubleshoot service/application connectivity issues and performance issues. You can start a packet capture in the web console Service Graph, or using the CLI. + +Packet capture integration with **Service Graph** makes it very easy to capture traffic for a specific namespace, service, replica set, daemonset, statefulset, or pod. Just right-click on an endpoint to start or schedule a capture, and then download capture files to your favorite visualization tool like Wireshark. + +With $[prodname] packet capture you can: + +- Run packet capture whenever you want (available 24/7) +- Preschedule packet captures to start and stop when needed +- Customize packet captures by port and protocol +- Share packet capture jobs + +**Demos and blogs** + +- [Video: packet capture demo](https://www.tigera.io/features/packet-capture/) +- [Troubleshooting microservices with Dynamic Packet Capture](https://thenewstack.io/faster-troubleshooting-with-dynamic-packet-capture/) + +## Concepts + +## About packet capture + +Typically, when you troubleshoot microservices and applications for connectivity issues or slow performance, you run a traditional packet capture tool like **tcpdump** against a container in a pod. But live troubleshooting in an ephemeral Kubernetes environment is tricky; problems do not last a long time, and happen randomly. So you need to be very fast to capture meaningful information to determine root causes. $[prodname] makes it easy with these basic steps: + +1. Determine the workload(s) you want to capture. +1. Start/schedule a packet capture job in Service Graph (Manager UI) or the CLI. +1. After the capture is finished, download the packet capture files (known as `pcap` files), and import them into your analysis tool (for example, Wireshark). + +For a simple use case workflow, see [Faster troubleshooting of microservices, containers, and Kubernetes with Dynamic Packet Capture](https://www.tigera.io/blog/faster-troubleshooting-of-microservices-containers-and-kubernetes-with-dynamic-packet-capture/). + +## Before you begin + +**Not supported** + +- Capturing traffic from host networked pods or host endpoints +- Capturing traffic from pods with multiple interfaces +- Capturing traffic for pods running on Windows hosts + +## How to + +- [Enable packet capture](#enable-packet-capture) +- [Packet capture in Service Graph](#packet-capture-in-service-graph) +- [Packet capture using the command line](#packet-capture-using-the-command-line) +- [Store and rotate capture files](#store-and-rotate-capture-files) +- [Enforce RBAC for capture tasks for CLI users](#enforce-rbac-for-capture-tasks-for-cli-users) + +### Enable packet capture + +##### Enable packet capture using kubectl + +Create a PacketCaptureAPI custom resource, named `tigera-secure` in the cluster. + +```bash +kubectl apply -f - < + +### Packet capture in Service Graph + +1. Select an endpoint from the service graph (for example, namespace, service, replica set, daemonset, statefulset, or pod), right-click, and select **Initiate packet capture**. + + ![start-capture](/img/calico-enterprise/start-capture.png) + +1. Schedule the capture to run now or at a later time, and click **Run**. + + ![schedule-pc](/img/calico-enterprise/schedule-pc.png) + +1. From the **Capture Jobs** tab in the bottom panel, the Status field will show that status, "Capturing". Scroll to the right, and click the drop-down menu for options to stop and manage captures. + + ![capture-menu](/img/calico-enterprise/capture-menu.png) + +### Packet capture using the command line + +This section provides examples of using the CLI to manage packet capture jobs and pcap files. + +**Create a PacketCapture resource** + +**Example: All pods in a namespace** + +This example captures traffic for all pods in the `sample` namespace. + +```yaml +apiVersion: projectcalico.org/v3 +kind: PacketCapture +metadata: + name: sample-capture-all + namespace: sample +spec: + selector: all() +``` + +**Example: All pods in a namespace matching a label and selector** + +This example captures traffic on all pods in the namespace `sample`, with the label `k8s-app`, equal to `nginx`. + +```yaml +apiVersion: projectcalico.org/v3 +kind: PacketCapture +metadata: + name: sample-capture-nginx + namespace: sample +spec: + selector: k8s-app == "nginx" +``` + +**Example: All pods in a namespace, TCP traffic only** + +This example captures traffic on all pods in the `sample` namespace, but only for TCP traffic. + +```yaml +apiVersion: projectcalico.org/v3 +kind: PacketCapture +metadata: + name: sample-capture-all + namespace: sample +spec: + selector: all() + filters: + - protocol: TCP +``` + +**Start a packet capture job, now** + +To start a packet capture job immediately, use the following command: + +```bash +kubectl apply -f +``` + +**Schedule a packet capture job** + +You can schedule a packet capture job to start and/or stop at a specific time using RFC3339 format. In the following example, a traffic capture job is scheduled for 10 minutes, between 00:30 UTC and 00:40 UTC for all pods in the sample namespace. + +```yaml +apiVersion: projectcalico.org/v3 +kind: PacketCapture +metadata: + name: sample-capture-all + namespace: sample +spec: + selector: all() + startTime: '2021-09-08T00:30:00Z' + endTime: '2021-09-08T00:40:00Z' +``` + +**Monitor status of packet capture job** + +After you start capture a job, it cycles through these states: Scheduled (if applicable), WaitingForTraffic, Capturing, and Finished. To monitor the status of a PacketCapture, use the following command: + +```bash +kubectl get packetcaptures -A +``` + +**Stop a packet capture job** + +To stop a capture job immediately, update the PacketCaptureResource by setting the `endTime` to the current time (or earlier). + +**Stop a packet capture job, and delete the capture file from the cluster** + +```bash +kubectl delete -f +``` + +**Delete a packet capture job** + +```bash +kubectl delete -f +``` + +**Find packet capture files** + +To find generated capture files, query the status of the PacketCapture: + +```bash +kubectl get packetcaptures -n -o yaml +``` + +```bash +export NS= +export NAME= +``` + +**Sample output** + +```yaml +apiVersion: projectcalico.org/v3 +kind: PacketCapture +metadata: + name: sample-capture-all + namespace: sample +spec: + selector: all() +status: + files: + - directory: /var/log/calico/pcap + fileNames: + - pod_cali.pcap + node: node-0 + state: Capturing +``` + +**Get packet capture files from pods** + +Get the pod on the node with the packet capture that you want. + +```bash +kubectl get pods -n tigera-fluentd --no-headers --field-selector spec.nodeName="" +``` + +Copy the packet capture using the pod information. + +```bash +kubectl cp tigera-fluentd/:var/log/calico/pcap/sample/sample-capture/ . +``` + +**Delete packet capture files** + +```bash +kubectl exec -it tigera-fluentd/ -- sh -c "rm -r /var/log/calico/pcap/sample/sample-capture/" +``` + +### Store and rotate capture files + +Packet capture files are stored on the host-mounted volume used for calico nodes. FelixConfig contains several parameters for storing and rotating capture files. + +**Note**: + +- Capture files are stored using the following directory structure: + `{namespace}/{packet capture resource name}` +- The active packet capture file is identified using the following schema: + `{workload endpoint name}_{host network interface}.pcap` +- Rotated capture file names contain an index matching the rotation timestamp +- Packet capture files are deleted after the packet capture resource is deleted. + +**Rotate capture files** + +The Felix parameter, `captureRotationSeconds` lets you schedule how often saved pcap are rotated. In the following example, the time rotation time is one day. + +```bash +kubectl patch felixconfiguration default -p '{"spec":{"captureRotationSeconds":"86400"}}' +``` + +### Enforce RBAC for capture tasks for CLI users + +Packet capture permissions are enforced using the standard Kubernetes RBAC for CLI users, based on Role and RoleBindings within a namespace. + +**Example** + +The following Role and RoleBindings shows how to allow user jane to create/delete/get/list/update/watch packet captures for a specific namespace. + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: sample + name: tigera-packet-capture-role +rules: + - apiGroups: ['projectcalico.org'] + resources: ['packetcaptures'] + verbs: ['get', 'list', 'watch', 'create', 'update', 'patch', 'delete'] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: tigera-packet-capture-role-jane + namespace: sample +subjects: + - kind: ServiceAccount + name: jane +roleRef: + kind: Role + name: tigera-packet-capture-role + apiGroup: rbac.authorization.k8s.io +``` + +To allow user jane to access (get and delete) the capture files generated for a specific namespace, a role/role binding similar to the one below can be used: + +```yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: tigera-authentication-clusterrole-jane +rules: + - apiGroups: ['projectcalico.org'] + resources: ['authenticationreviews'] + verbs: ['create'] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: tigera-authentication-clusterrolebinding-jane +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: tigera-authentication-clusterrole-jane +subjects: + - kind: ServiceAccount + name: jane + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + namespace: sample + name: tigera-capture-files-role +rules: + - apiGroups: ['projectcalico.org'] + resources: ['packetcaptures/files'] + verbs: ['get', 'delete'] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: tigera-capture-files-role-jane + namespace: sample +subjects: + - kind: ServiceAccount + name: jane + namespace: default +roleRef: + kind: Role + name: tigera-capture-files-role + apiGroup: rbac.authorization.k8s.io +``` diff --git a/calico-cloud_versioned_docs/version-24-1/observability/visualize-traffic.mdx b/calico-cloud_versioned_docs/version-24-1/observability/visualize-traffic.mdx new file mode 100644 index 0000000000..a5dc6462dc --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/observability/visualize-traffic.mdx @@ -0,0 +1,9 @@ +--- +description: Visualize cluster traffic to and from external endpoints in Calico Cloud Service Graph with network sets that group external IP ranges by purpose. +--- + +# Visualize traffic to and from a cluster + +With Service Graph, you know the value of seeing pod-to-pod traffic within your cluster. But what about traffic external to your cluster? To learn how to visualize traffic to and from a cluster, see [Get started with network sets](../network-policy/networksets.mdx). + +![google-networkset](/img/calico-enterprise/google-networkset.png) diff --git a/calico-cloud_versioned_docs/version-24-1/operations/cluster-management.mdx b/calico-cloud_versioned_docs/version-24-1/operations/cluster-management.mdx new file mode 100644 index 0000000000..f85dc243b7 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/cluster-management.mdx @@ -0,0 +1,61 @@ +--- +description: Manage Calico Cloud connected clusters from the web console by adding clusters, removing managed clusters, and configuring cluster-wide alerts. +--- + +# Manage clusters + +## Connect new clusters + +To connect new clusters to $[prodname], select Managed Clusters in the left nav, and click **Connect Cluster**. + +## Remove a cluster from the managed clusters list + +### Prerequisites + +- You are signed in to the web console and you have [Owner or Admin privileges](../users/user-management.mdx). +- The cluster you want to remove is [disconnected from $[prodname]](disconnect.mdx). + +### Remove a cluster + +For the cluster you want to remove, select **Actions** > **Delete**, and then confirm your choice by selecting **Delete cluster**. + +:::note + +Deleting a managed cluster does not remove $[prodname] logs. + +If a new managed cluster is connected with the same name as a previously deleted managed cluster, the logs associated with the deleted managed cluster will be displayed as if they belong to the new managed cluster. + +::: + +### Recover a deleted managed cluster + +If you inadvertently delete a managed cluster, you may restore it by connecting your cluster using the same name as the previously deleted cluster. For help with connecting a managed cluster see [Connect a cluster to $[prodname]](../get-started/install-cluster.mdx). + +## Manage and monitor multiple clusters in the web console + +### Access your managed clusters + +Each time you install $[prodname] on a cluster, the cluster is added to the drop-down menu in the web console. + +![cluster-menu](/img/calico-cloud/cluster-menu.png) + +Use this menu to move between your clusters. You can connect as many managed clusters as your subscriptions allows. + +### Managed cluster views + +When you select a different cluster from the cluster menu, the entire web console view changes to reflect the selected cluster. +For example, each managed cluster has unique indexes for Elasticsearch clusters, and the dashboard reflects only the selected managed cluster. + +### Manage policies in multiple clusters + +To implement, modify, or view network policies in a given cluster: + +1. Toggle to a managed cluster using the managed cluster drop-down menu. +1. Define, modify or view network policies in that cluster. +1. You can monitor the impact of a policy using the policy visualization tool (Flow Viz). + +To apply policies in one managed cluster to a different cluster, manually apply the manifest/.yaml using `kubectl`. + +To download manifests for a policy, click **Export** for the selected policy. + +![export-policy](/img/calico-cloud/export-policy.png) diff --git a/calico-cloud_versioned_docs/version-24-1/operations/comms/index.mdx b/calico-cloud_versioned_docs/version-24-1/operations/comms/index.mdx new file mode 100644 index 0000000000..e5c544dd28 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/comms/index.mdx @@ -0,0 +1,144 @@ +--- +description: Provide custom TLS certificates for Calico components. +--- + +# Provide TLS certificates for Calico components + +$[prodname] uses TLS certificates for mutual authentication between components. The operator automatically generates and manages these certificates using a self-signed CA (`tigera-operator-signer`). + +To replace any certificate with your own, create a Kubernetes secret with the same name in the `tigera-operator` namespace. The operator will detect it and use your certificate instead of the auto-generated one. + +## Certificate requirements + +- **Extended Key Usages**: include both `TLS Web Server Authentication` and `TLS Web Client Authentication` +- **Common Name (CN)**: must match the first DNS name listed in the table below. Note that `node-certs` and `typha-certs` use `typha-client` and `typha-server` respectively (not the component name). +- **Subject Alternative Names (SANs)**: include all DNS names listed for the secret + +## Create or update a secret + +```bash +SIGNER=my-ca-signer +kubectl create secret generic -n tigera-operator \ + --from-file=tls.crt= \ + --from-file=tls.key= && \ +kubectl label secret -n tigera-operator operator.tigera.io/signer=$SIGNER +``` + +To update an existing secret: + +```bash +SIGNER=my-ca-signer +kubectl create secret generic -n tigera-operator \ + --from-file=tls.crt= \ + --from-file=tls.key= \ + --dry-run=client -o yaml | kubectl replace -f - && \ +kubectl label secret -n tigera-operator operator.tigera.io/signer=$SIGNER --overwrite +``` + +:::note + +Updating a certificate causes the affected components to restart. Expect brief unavailability during the rollout. + +::: + +## TLS certificate reference + +The **Deployed to** column shows the namespace where the operator places the secret at runtime. To override, always create your secret in `tigera-operator`. + +| Secret name | DNS name(s) | Deployed to | Owner resource | +|---|---|---|---| +| `calico-apiserver-certs` | `calico-api` | `calico-system` | APIServer/tigera-secure | +| `calico-kube-controllers-metrics-tls` | `calico-kube-controllers-metrics` | `calico-system` | Installation/default | +| `calico-node-prometheus-client-tls` | `calico-node-prometheus-client-tls` | `tigera-prometheus` | Monitor/tigera-secure | +| `calico-node-prometheus-server-tls` | `calico-node-metrics` | `calico-system` | Installation/default | +| `calico-node-prometheus-tls` | `prometheus-http-api` | `tigera-prometheus` | Monitor/tigera-secure | +| `deep-packet-inspection-tls` | `intrusion-detection-tls` | `tigera-dpi` | IntrusionDetection/tigera-secure | +| `node-certs` | `typha-client` | `calico-system` | Installation/default | +| `node-certs` | `typha-client` | `tigera-dpi` | IntrusionDetection/tigera-secure | +| `tigera-ee-elasticsearch-metrics-tls` | `tigera-elasticsearch-metrics` | `tigera-elasticsearch` | LogStorage/tigera-secure | +| `tigera-fluentd-prometheus-tls` | `fluentd-http-input` | `tigera-fluentd` | LogCollector/tigera-secure | +| `tigera-operator-tls` | `tigera-operator-metrics` | `tigera-prometheus` | Monitor/tigera-secure | +| `typha-certs` | `typha-server` | `calico-system` | Installation/default | +| `typha-certs-noncluster-host` | `typha-server-noncluster-host` | `calico-system` | Installation/default | + +:::tip + +Typha and Node use mutual TLS. If you replace `typha-certs`, `typha-certs-noncluster-host`, or `node-certs`: +- Ensure they are all signed by the same CA. Mismatched certificates will break Node-to-Typha communication. +- These secrets require an additional `common-name` data field containing the CN. For example: + ```bash + kubectl create secret generic typha-certs -n tigera-operator \ + --from-file=tls.crt= \ + --from-file=tls.key= \ + --from-literal=common-name=typha-server + ``` + +::: + +## Monitor certificates + +The operator labels and annotates every TLS secret it manages: + +- **Label** `certificates.operator.tigera.io/signer` — the signer that issued the certificate +- **Annotation** `certificates.operator.tigera.io/issuer` — the issuer name +- **Annotation** `certificates.operator.tigera.io/expiry` — certificate expiration timestamp + +List all managed certificates with their issuer and expiry: + +```bash +kubectl get secrets -A -l certificates.operator.tigera.io/signer -o json | \ + jq -r '.items[] | select(.metadata.namespace != "tigera-operator") | + [.metadata.namespace, .metadata.name, + .metadata.annotations["certificates.operator.tigera.io/issuer"], + .metadata.annotations["certificates.operator.tigera.io/expiry"]] | @tsv' | \ + column -t -N NAMESPACE,NAME,ISSUER,EXPIRY +``` + +``` +NAMESPACE NAME ISSUER EXPIRY +calico-system calico-apiserver-certs tigera-operator-signer 2028-07-12T17:59:28Z +calico-system manager-tls tigera-operator-signer 2028-07-12T18:01:21Z +calico-system node-certs tigera-operator-signer 2028-07-12T17:59:27Z +calico-system typha-certs tigera-operator-signer 2028-07-12T17:59:27Z +tigera-elasticsearch tigera-secure-linseed-cert tigera-operator-signer 2028-07-12T17:59:23Z +... +``` + +The operator also exposes Prometheus metrics for certificate expiration. See [Operator metrics](../monitor/metrics/operator-metrics.mdx) to configure monitoring and alerts. + +## Certificate management with Kubernetes CSR API + +Instead of providing certificates directly, you can configure $[prodname] to use the [Kubernetes Certificates API](https://kubernetes.io/docs/reference/access-authn-authz/certificate-signing-requests/) for automated certificate issuance. + +### Enable certificate management + +Add the `certificateManagement` section to your Installation resource: + +```yaml +apiVersion: operator.tigera.io/v1 +kind: Installation +metadata: + name: default +spec: + certificateManagement: + caCert: + signerName: / + signatureAlgorithm: SHA512WithRSA + keyAlgorithm: RSAWithSize4096 +``` + +**Supported algorithms:** +- Private key: RSA (2048, 4096, 8192), ECDSA (256, 384, 521) +- Signature: RSA-SHA (256, 384, 512), ECDSA-SHA (256, 384, 512) + +### How it works + +Pods use an init container to create a CertificateSigningRequest (CSR). The pod remains in `Init` state until the CSR is approved and signed by your certificate authority. + +Monitor CSRs: + +```bash +kubectl get csr -w +``` + +CSR names follow the pattern: `::`. diff --git a/calico-cloud_versioned_docs/version-24-1/operations/component-logs.mdx b/calico-cloud_versioned_docs/version-24-1/operations/component-logs.mdx new file mode 100644 index 0000000000..383061d95c --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/component-logs.mdx @@ -0,0 +1,121 @@ +--- +description: Reference for locating and collecting Calico Cloud component logs including calico/node, Felix, Typha, and observability agents on connected clusters. +--- + +# Component logs + +## Big picture + +View and collect $[prodname] logs. + +## Value + +It is useful to view logs to monitor component health and diagnose potential issues. + +## Concepts + +### $[nodecontainer] logs + +The $[nodecontainer] logs contain log output from the following subcomponents: + +- Per-node startup logic +- BGP agent +- Felix policy agent + +Components log either to disk within `/var/log/calico`, to stdout, or both. + +For components that log to disk, files are automatically rotated, and by default 10 files of 1MB each are kept. The current log file is called `current` and rotated files have @ followed by a timestamp detailing when the files was rotated in [tai64n](http://cr.yp.to/libtai/tai64.html#tai64n) format. + +## How to + +## View logs for a $[nodecontainer] instance + +You can view logs for a node using the `kubectl logs` command. This will show logs for all subcomponents of the given node. + +For example: + +``` +kubectl logs -n calico-system calico-node-xxxx +``` + +## View logs from the CNI plugin + +CNI plugin logs are not available through kubectl and are instead logged both to the host machine's disk as well as stderr. + +By default, these logs can be found at `/var/log/calico/cni/` on the host machine. + +The container runtime may also display the CNI plugin logs within its own log output. + +## Configure BGP agent log level + +BGP log level is configured via the [BGPConfiguration](../reference/resources/bgpconfig.mdx) API, and can be one of the following values: + +- `Debug`: enables "debug all" logging for BIRD. The most verbose logging level. +- `Info`: enables logging for protocol state changes. This is the default log level. +- `Warning`: disables BIRD logging, emits warning level configuration logs only. +- `Error`: disables BIRD logging, emits error level configuration logs only. +- `Fatal`: disables BIRD logging, emits fatal level configuration logs only. + +To modify the BGP log level: + +1. Get the current bgpconfig settings. + + ```bash + kubectl get bgpconfiguration.projectcalico.org -o yaml > bgp.yaml + ``` + +1. Modify logSeverityScreen to the desired value. + + ```bash + vim bgp.yaml + ``` + + :::tip + + For a global change set the name to "default". + For a node-specific change set the name to the node name prefixed with "node.", e.g., "node.node-1". + + ::: + +1. Replace the current bgpconfig settings. + + ```bash + kubectl replace -f bgp.yaml + ``` + +## Configure Felix log level + +Felix log level is configured via the [FelixConfiguration](../reference/resources/felixconfig.mdx) API, and can be one of the following values: + +- `Debug`: The most verbose logging level - for development and debugging. +- `Info`: The default log level. Shows important state changes. +- `Warning`: Shows warnings only. +- `Error`: Shows errors only. +- `Fatal`: Shows fatal errors only. + +To modify Felix's log level: + +1. Get the current felixconfig settings. + + ```bash + kubectl get felixconfiguration.projectcalico.org default -o yaml > felix.yaml + ``` + +1. Modify logSeverityScreen to desired value. + + ```bash + vim felixconfig.yaml + ``` + + :::tip + + For a global change set the name to "default". + For a node-specific change set the name to the node name prefixed with "node.", e.g., "node.node-1". + + ::: + +1. Replace the current felixconfig settings. + + ``` + kubectl replace -f felixconfig.yaml + ``` diff --git a/calico-cloud_versioned_docs/version-24-1/operations/disconnect.mdx b/calico-cloud_versioned_docs/version-24-1/operations/disconnect.mdx new file mode 100644 index 0000000000..b5d58615d4 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/disconnect.mdx @@ -0,0 +1,55 @@ +--- +description: Disconnect a cluster from Calico Cloud with the migration script that uninstalls SaaS components and reverts the cluster to open-source Project Calico. +--- + +# Uninstall Calico Cloud from a cluster + +Whether you've finished with your $[prodname] Trial or decided to disconnect your cluster from +$[prodname], we know you want your cluster to remain functional. We highly recommend running +a simple script to migrate your cluster to open-source Project Calico. + +## About the migration script + +The script migrates all applicable $[prodname] components to open-source Project Calico; this includes +removal and cleanup of all $[prodname] components that have no equivalents in Project Calico. +Because Project Calico does not have the tier resource, the script will exit if any policies +exist in any tier except for the `default` or `calico-system` tiers. +To remove policies from tiers, you have these options: + +- Manually move policies out of tiers prior to running the script +- Let the script remove _ALL_ Calico policies by specifying the `--remove-all-calico-policy` flag + +:::note + +To successfully downgrade to an open-source Calico configuration, policies must allow necessary traffic +to and from $[prodname] and open-source Calico namespaces. If you keep policies in the default tier +(especially GlobalNetworkPolicies), and you have default deny policies, update or add policies accordingly +to allow this necessary traffic. + +::: + +:::warning + +If your cluster began with Calico installed and managed by AKS with the `addon-manager` component, this uninstall process will +not be successful. You will need to reach out to your support contact to create a plan to uninstall $[prodname]. + +::: + +### Before you begin + +* You have `kubectl` administrator access to the cluster you want to migrate to Calico Open Source. +* You are accessing the cluster from a Linux-based machine. + +### Run the migration script + +1. Download the script `curl -O $[clouddownloadurl]/downgrade.sh`. + +1. Make the script executable `chmod +x downgrade.sh`. + +1. Run the script and read the help to determine if you need to specify any flags `./downgrade.sh --help`. + +1. Run the script with any needed flags, for example: `./downgrade.sh --remove-prometheus`. + +## Next steps + +Continue using your cluster with open-source [Project Calico](/calico/latest/about). diff --git a/calico-cloud_versioned_docs/version-24-1/operations/ebpf/enabling-ebpf.mdx b/calico-cloud_versioned_docs/version-24-1/operations/ebpf/enabling-ebpf.mdx new file mode 100644 index 0000000000..e290fd8c83 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/ebpf/enabling-ebpf.mdx @@ -0,0 +1,300 @@ +--- +description: Switch a Calico Cloud connected cluster to the eBPF data plane on an existing installation as an alternative to the iptables data plane. +--- + +# Enable eBPF on an existing cluster + +import EbpfValue from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_ebpf-value.mdx'; + +## Big picture + +Enable the eBPF data plane on an existing cluster. + +## Value + + + +## Concepts + +### eBPF + +eBPF (or "extended Berkeley Packet Filter"), is a technology that allows safe mini programs to be attached to various low-level hooks in the Linux kernel. eBPF has a wide variety of uses, including networking, security, and tracing. You’ll see a lot of non-networking projects leveraging eBPF, but for $[prodname] our focus is on networking, and in particular, pushing the networking capabilities of the latest Linux kernels to the limit. + +## Before you begin + +**Supported architecture and versions** + +- x86-64 +- arm64 (little-endian) +- Linux distribution/kernel: + + - Ubuntu 22.04 or above. + - Red Hat v8.4 with Linux kernel v4.18.0-305 or above (Red Hat have backported the required features to that build). + - Another supported distribution with Linux kernel v5.10 or above. {/*TODO-XREFS-CC */} + - Immutable operating systems (e.g., Talos Linux): Ensure the `CgroupV2Path` in the `FelixConfiguration` CRD is set to a writable path. + +#### Kernel version requirements for eBPF features + +Some eBPF features require a higher kernel version than the base eBPF data plane. + +| Feature | Minimum kernel version | Details | +|---|---|---| +| Base eBPF data plane | v5.10 (RHEL: v4.18.0-305) | CO-RE supported at this version for better performance | +| Log rules in eBPF mode | v5.16 | Logs sent to trace pipe via `bpftool prog tracelog` | +| [QoS bandwidth controls](../../networking/configuring/qos-controls.mdx) | v6.6 | Requires `tcx` support. Established connection limits not supported with eBPF | +| [DNS policy inline mode](../../network-policy/domain-based-policy.mdx) | v5.17 (RHEL: v5.14) | `BPFDNSPolicyMode: Inline` parses DNS responses in eBPF before they reach the application. Only wildcard prefixes (`*.x.y.z`) supported. Falls back to `NoDelay` on older kernels | + +:::warning + +The minimum kernel version required for the eBPF data plane is v5.10, which includes support for [CO-RE (Compile Once - Run Everywhere)](https://docs.ebpf.io/concepts/core/). CO-RE significantly improves compatibility and performance across kernel versions. For access to all eBPF features, we recommend kernel v6.6 or above. + +::: + +- An underlying network fabric that allows VXLAN traffic between hosts. In eBPF mode, VXLAN is used to forward Kubernetes NodePort traffic. + +**Unsupported platforms** + +- GKE +- MKE +- TKG + +:::note + +eBPF supports AKS with Calico CNI and $[prodname] network policy. However, with AKS with Azure CNI and $[prodname] network policy, kube-proxy cannot be disabled so the performance benefits of eBPF are lost. However, there are other reasons to use eBPF other than performance gains, as described in [eBPF use cases](use-cases-ebpf.mdx). +{/*TODO-XREFS-CC */} + +::: + +**Unsupported features** + +- Clusters with some eBPF nodes and some standard data plane and/or Windows nodes +- IPv6 +- Host endpoint `doNotTrack` policy (other policy types are supported) +- Floating IPs +- SCTP (either for policy or services) +- Tagged VLAN devices +- L7 logs +- Application layer policies +- Web application firewall (WAF) + +**Recommendations for performance** + +For best pod-to-pod performance, we recommend using an underlying network that doesn't require $[prodname] to use an overlay. For example: + +- A cluster within a single AWS subnet +- A cluster using a compatible cloud provider's CNI (such as the AWS VPC CNI plugin) +- An on-prem cluster with BGP peering configured + +If you must use an overlay, we recommend that you use VXLAN, not IPIP. VXLAN has better performance than IPIP in eBPF mode due to various kernel optimizations. + +## How to + +- [Verify that your cluster is ready for eBPF mode](#verify-that-your-cluster-is-ready-for-ebpf-mode) +- [Configure $[prodname] to talk directly to the API server](#configure-calico-cloud-to-talk-directly-to-the-api-server) +- [Configure kube-proxy](#configure-kube-proxy) +- [Enable eBPF mode](#enable-ebpf-mode) +- [Try out DSR mode](#try-out-direct-server-return-mode) +- [Reversing the process](#reversing-the-process) + +### Verify that your cluster is ready for eBPF mode + +This section explains how to make sure your cluster is suitable for eBPF mode. + +To check that the kernel on a node is suitable, you can run + +```bash +uname -rv +``` + +The output should look like this: + +``` +5.10.0-26-generic #28~20.04.1-Ubuntu SMP Fri Jan 27 14:30:10 UTC 2023 +``` + +In this case the kernel version is v5.10, which is suitable. + +On Red Hat-derived distributions, you may see something like this: + +``` +4.18.0-305.el8.x86_64 (mockbuild@x86-vm-08.build.eng.bos.redhat.com) +``` + +Since the Red Hat kernel is v4.18 with at least build number 305 (RHEL 8.4), this kernel is suitable. + +### Configure $[prodname] to talk directly to the API server + +In eBPF mode, $[prodname] implements Kubernetes service networking directly (rather than relying on `kube-proxy`). +Of course, this makes it highly desirable to disable `kube-proxy` when running in eBPF mode to save resources +and avoid confusion over which component is handling services. + +To be able to disable `kube-proxy`, $[prodname] needs to communicate to the API server _directly_ rather than +going through `kube-proxy`. To make _that_ possible, we need to find a persistent, static way to reach the API server. +The best way to do that varies by Kubernetes distribution: + +- If you created a cluster manually (for example by using `kubeadm`) then the right address to use depends on whether you + opted for a high-availability cluster with multiple API servers or a simple one-node API server. + + - If you opted to set up a high availability cluster then you should use the address of the load balancer that you + used in front of your API servers. As noted in the Kubernetes documentation, a load balancer is required for a + HA set-up but the precise type of load balancer is not specified. + - If you opted for a single control plane node then you can use the address of the control plane node itself. However, + it's important that you use a _stable_ address for that node such as a dedicated DNS record, or a static IP address. + If you use a dynamic IP address (such as an EC2 private IP) then the address may change when the node is restarted + causing $[prodname] to lose connectivity to the API server. + +- `kops` typically sets up a load balancer of some sort in front of the API server. You should use + the FQDN and port of the API load balancer, for example `api.internal.` as the `KUBERNETES_SERVICE_HOST` + below and 443 as the `KUBERNETES_SERVICE_PORT`. +- OpenShift requires various DNS records to be created for the cluster; one of these is exactly what we need: + `api-int..` should point to the API server or to the load balancer in front of the + API server. Use that (filling in the `` and `` as appropriate for your cluster) for the + `KUBERNETES_SERVICE_HOST` below. OpenShift uses 6443 for the `KUBERNETES_SERVICE_PORT`. +- For AKS and EKS clusters you should use the FQDN of the API server's load balancer. This can be found with + ``` + kubectl cluster-info + ``` + which gives output like the following: + ``` + Kubernetes master is running at https://60F939227672BC3D5A1B3EC9744B2B21.gr7.us-west-2.eks.amazonaws.com + ... + ``` + In this example, you would use `60F939227672BC3D5A1B3EC9744B2B21.gr7.us-west-2.eks.amazonaws.com` for + `KUBERNETES_SERVICE_HOST` and `443` for `KUBERNETES_SERVICE_PORT` when creating the config map. + +Once you've found the correct address for your API server, create the following config map in the `tigera-operator` +namespace using the host and port that you found above: + +```yaml +kind: ConfigMap +apiVersion: v1 +metadata: + name: kubernetes-services-endpoint + namespace: tigera-operator +data: + KUBERNETES_SERVICE_HOST: '' + KUBERNETES_SERVICE_PORT: '' +``` + +The operator will pick up the change to the config map automatically and do a rolling update of $[prodname] to pass on the change. Confirm that pods restart and then reach the `Running` state with the following command: + +``` +watch kubectl get pods -n calico-system +``` + +If you do not see the pods restart then it's possible that the `ConfigMap` wasn't picked up (sometimes Kubernetes is slow to propagate `ConfigMap`s (see Kubernetes [issue #30189](https://github.com/kubernetes/kubernetes/issues/30189))). You can try restarting the operator. + +### Configure kube-proxy + +In eBPF mode $[prodname] replaces `kube-proxy` so it wastes resources (and reduces performance) to run both. +This section explains how to disable `kube-proxy` in some common environments. + +#### Clusters that run `kube-proxy` with a `DaemonSet` (such as `kubeadm`) + +For a cluster that runs `kube-proxy` in a `DaemonSet` (such as a `kubeadm`-created cluster), you can disable `kube-proxy` reversibly by adding a node selector to `kube-proxy`'s `DaemonSet` that matches no nodes, for example: + +``` +kubectl patch ds -n kube-system kube-proxy -p '{"spec":{"template":{"spec":{"nodeSelector":{"non-calico": "true"}}}}}' +``` + +Then, should you want to start `kube-proxy` again, you can simply remove the node selector. + +:::note + +This approach is not suitable for AKS with Azure CNI since that platform makes use of the Kubernetes add-on manager. +the change will be reverted by the system. For AKS, you should follow [Avoiding conflicts with kube-proxy](#avoiding-conflicts-with-kube-proxy) +below. + +::: + +#### OpenShift + +If you are running OpenShift, you can disable `kube-proxy` as follows: + +``` +kubectl patch networks.operator.openshift.io cluster --type merge -p '{"spec":{"deployKubeProxy": false}}' +``` + +To re-enable it: + +``` +kubectl patch networks.operator.openshift.io cluster --type merge -p '{"spec":{"deployKubeProxy": true}}' +``` + +### Avoiding conflicts with kube-proxy + +If you cannot disable `kube-proxy` (for example, because it is managed by your Kubernetes distribution), then you _must_ change Felix configuration parameter `BPFKubeProxyIptablesCleanupEnabled` to `false`. This can be done with `kubectl` as follows: + +``` +kubectl patch felixconfiguration default --patch='{"spec": {"bpfKubeProxyIptablesCleanupEnabled": false}}' +``` + +If both `kube-proxy` and `BPFKubeProxyIptablesCleanupEnabled` is enabled then `kube-proxy` will write its iptables rules and Felix will try to clean them up resulting in iptables flapping between the two. + +### Enable eBPF mode + +To enable eBPF mode, change the `spec.calicoNetwork.linuxDataplane` parameter in the operator's `Installation` +resource to `"BPF"`. + +```bash +kubectl patch installation.operator.tigera.io default --type merge -p '{"spec":{"calicoNetwork":{"linuxDataplane":"BPF"}}}' +``` + +When enabling eBPF mode, preexisting connections keep using the standard data plane. They do not benefit +from eBPF mode's advantages, and whether they survive the switch depends on the protocol: + +- **TCP connections continue to work**, including any address translation applied to them. The translation + lives in the Linux conntrack entry rather than in the `kube-proxy` rules, so a connection through a + service keeps reaching the backend it was already using. +- **Traffic that is not TCP is treated as new.** eBPF mode has no record of an established UDP flow, so it + applies policy and picks a service backend again. A UDP flow to a service with several backends can move + to a different backend, and a flow whose reply direction was allowed only because the connection was + already established is dropped. +- **Connections over a VXLAN overlay do not survive.** $[prodname] recreates the VXLAN device when it + switches to eBPF mode, and the kernel drops the conntrack state for the flows through it. + +:::note + +The operator rolls out the change with a rolling update (non-disruptive) and then swiftly transitions all nodes to eBPF mode. However, it's inevitable that some nodes will enter eBPF mode before others. This can disrupt the flow of traffic through node ports. + +::: + +### Try out direct server return mode + +Direct server return (DSR) mode skips a hop through the network for traffic to services (such as node ports) from outside the cluster. +This reduces latency and CPU overhead but it requires the underlying network to allow nodes to send traffic with each other's IPs. + +In AWS, this requires all your nodes to be in the same subnet and for the source/dest check to be disabled. +In GCP, the source/dest check should also be disabled, which can be done by enabling IP forwarding. + +DSR mode is disabled by default; to enable it, set the `BPFExternalServiceMode` Felix configuration parameter to `"DSR"`. This can be done with `kubectl`: + +``` +kubectl patch felixconfiguration default --patch='{"spec": {"bpfExternalServiceMode": "DSR"}}' +``` + +To switch back to tunneled mode, set the configuration parameter to `"Tunnel"`: + +``` +kubectl patch felixconfiguration default --patch='{"spec": {"bpfExternalServiceMode": "Tunnel"}}' +``` + +Switching external traffic mode can disrupt in-progress connections. + +### Reversing the process + +To revert to standard Linux networking: + +1. Reverse the changes to the operator's `Installation`: + + ```bash + kubectl patch installation.operator.tigera.io default --type merge -p '{"spec":{"calicoNetwork":{"linuxDataplane":"Iptables"}}}' + ``` + +1. If you disabled `kube-proxy`, re-enable it (for example, by removing the node selector added above). + + ``` + kubectl patch ds -n kube-system kube-proxy --type merge -p '{"spec":{"template":{"spec":{"nodeSelector":{"non-calico": null}}}}}' + ``` + +1. Since disabling eBPF mode is disruptive to existing connections, monitor existing workloads to make sure they re-establish any connections that were disrupted by the switch. diff --git a/calico-cloud_versioned_docs/version-24-1/operations/ebpf/index.mdx b/calico-cloud_versioned_docs/version-24-1/operations/ebpf/index.mdx new file mode 100644 index 0000000000..5fdfbb2534 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/ebpf/index.mdx @@ -0,0 +1,11 @@ +--- +description: Reference index for the Calico Cloud eBPF data plane on connected clusters covering installation, runtime switching, troubleshooting, and use-case selection. +hide_table_of_contents: true +--- + +# eBPF data plane mode + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/operations/ebpf/troubleshoot-ebpf.mdx b/calico-cloud_versioned_docs/version-24-1/operations/ebpf/troubleshoot-ebpf.mdx new file mode 100644 index 0000000000..a51b2046fb --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/ebpf/troubleshoot-ebpf.mdx @@ -0,0 +1,413 @@ +--- +description: Troubleshooting guide for the Calico Cloud eBPF data plane on a connected cluster covering verification logs, service connectivity, BPF map inspection, and failure modes. +--- + +# Troubleshoot eBPF mode + +This document gives general troubleshooting guidance for the eBPF data plane. + +To understand basic concepts, we recommend the following video by Tigera Engineers: [Opening the Black Box: Understanding and troubleshooting Calico's eBPF Data Plane](https://www.youtube.com/watch?v=Mh43sNBu208). + +## Troubleshoot access to services + +- Verify that eBPF mode is correctly enabled + + Examine the log for a `$[noderunning]` container; in the extremely rare case when + eBPF mode is not supported it will log an `ERROR` log that says + + ```bash + BPF data plane mode enabled but not supported by the kernel. Disabling BPF mode. + ``` + + If BPF mode is correctly enabled, you should see an `INFO` log that says + + ```bash + BPF enabled, starting BPF endpoint manager and map manager. + ``` + +- In eBPF mode, forwarding external client access to services (typically NodePorts) from node to node is implemented using VXLAN encapsulation. + If NodePorts time out when the backing pod is on another node, check your underlying network fabric allows + VXLAN traffic between the nodes. VXLAN is a UDP protocol; by default it uses port 4789. + + Note that this VXLAN traffic is separate from any overlay network that you may be using for pod-to-pod traffic. + +- In DSR mode, $[prodname] requires that the underlying network fabric allows one node to respond on behalf of + another. + + - In AWS, to allow this, the Source/Dest check must be disabled on the node's NIC. However, note that DSR only + works within AWS; it is not compatible with external traffic through a load balancer. This is because the load + balancer is expecting the traffic to return from the same host. + + - In GCP, the "Allow forwarding" option must be enabled. As with AWS, traffic through a load balancer does not + work correctly with DSR because the load balancer is not consulted on the return path from the backing node. + +# The `calico-node -bpf` tool + + To inspect $[prodname]'s internal data structures, you can use the `calico-node + -bpf` tool. The tool is embedded in the $[nodecontainer] container image and + displays information about the eBPF data plane from within a `calico-node` pod + only. Use `kubectl get pod -o wide -n calico-system` to find the name of a + `calico-node` pod and use the name in the following commands instead of + ``. + + To run the tool, use: + + ```bash + kubectl exec -n calico-system -- calico-node -bpf + ``` + + For example, to show the tool's help: + + ```bash + kubectl exec -n calico-system -- calico-node -bpf help + + Available Commands: + arp Manipulates arp + cleanup Removes all calico-bpf programs and maps + completion Generate the autocompletion script for the specified shell + connect-time Manipulates connect-time load balancing programs + conntrack Manipulates connection tracking + counters Show and reset counters + help Help about any command + ifstate Manipulates ifstate + ipsets Manipulates ipsets + nat Manipulates network address translation (nat) + policy Dump policy attached to interface + profiling Show and reset profiling data + routes Manipulates routes + version Prints the version and exits + + (Since the tool is embedded in the main `calico-node` binary the `--help` option is not available, but running + `calico-node -bpf help` does work.) + ``` + + For example, to dump the BPF conntrack table, use: + + ``` + kubectl exec -n calico-system -- calico-node -bpf conntrack dump + ... + ``` + +## Debug access to services + + Inspect the BPF NAT table to verify that the service is correctly programmed. To dump the BPF NAT table: + + ``` + kubectl exec -n calico-system -- calico-node -bpf nat dump + + 10.96.0.10 port 53 proto 6 id 4 count 2 local 0 + 4:0 192.168.129.66:53 + 4:1 192.168.129.68:53 + 10.96.0.10 port 53 proto 17 id 6 count 2 local 0 + 6:0 192.168.129.66:53 + 6:1 192.168.129.68:53 + 10.96.0.10 port 9153 proto 6 id 5 count 2 local 0 + 5:0 192.168.129.66:9153 + 5:1 192.168.129.68:9153 + 10.105.77.92 port 5473 proto 6 id 0 count 2 local 0 + 0:0 10.128.1.192:5473 + 0:1 10.128.1.195:5473 + 10.105.187.231 port 8081 proto 6 id 2 count 1 local 0 + 2:0 192.168.105.131:8081 + 10.109.136.88 port 7443 proto 6 id 1 count 1 local 0 + 1:0 192.168.129.72:7443 + 10.109.139.39 port 443 proto 6 id 7 count 2 local 0 + 7:0 192.168.129.67:5443 + 7:1 192.168.129.69:5443 + 10.96.0.1 port 443 proto 6 id 3 count 1 local 0 + 3:0 10.128.0.255:6443 + + ``` + + Inspect the BPF conntrack table to verify that connections are being tracked. To dump the BPF conntrack table: + + ``` + kubectl exec -n calico-system -- calico-node -bpf conntrack dump + + TCP 192.168.58.7:49178 -> 10.111.57.87:80 -> 192.168.105.136:80 Active ago 4.486606371s CLOSED <--- example of connection to service with per-packet NAT + TCP 10.128.1.194:41513 -> 10.128.1.192:179 Active ago 26.442759238s ESTABLISHED <--- example of connection without NAT or with connect-time NAT + TCP 192.168.58.7:42818 -> 10.111.57.87:80 -> 192.168.105.136:80 Active ago 1m15.987585857s CLOSED + TCP 10.128.1.192:58208 -> 10.109.136.88:7443 -> 192.168.58.5:7443 Active ago 4.935017508s ESTABLISHED + UDP 162.142.125.240:30603 -> 10.128.1.192:18989 Active ago 1m0.816678617s + UDP 127.0.0.1:48611 -> 127.0.0.53:53 Active ago 17.789851961s + ``` + + Note that traffic originating within the cluster uses connect-time load balancing. By default, the connect-time + load balancing is only enabled for TCP traffic. When connect-time load + balancing is used, the conntrack table will not show the NAT resolution as + that happens when the application calls `connect()`. + +## Check if $[prodname] is dropping packets + + If you suspect that $[prodname] is dropping packets, you can use the + `calico-node -bpf` tool to check the BPF counters. Since the eBPF data plane is split + into programs that are attached to interfaces, you must check the counters on + the relevant interface. You can either dump counters for all interfaces or use `--iface=` to dump + counters for a specific interface. + + Increasing counter `Dropped by policy` indicates that $[prodname] is dropping packets due to policy and you should + check your policy configuration. + + ``` + kubectl exec -n calico-system -- calico-node -bpf counters dump --iface=eth0 + +----------+--------------------------------+---------+--------+-----+ + | CATEGORY | TYPE | INGRESS | EGRESS | XDP | + +----------+--------------------------------+---------+--------+-----+ + | Accepted | by another program | 0 | 0 | N/A | + | | by failsafe | 0 | 0 | N/A | + | | by policy | 0 | 4 | N/A | + | Dropped | NAT source collision | 0 | 0 | N/A | + | | resolution failed | | | | + | | QoS control limit | 0 | 0 | N/A | + | | by policy | 0 | 11 | N/A | + | | failed decapsulation | 0 | 0 | N/A | + | | failed encapsulation | 0 | 0 | N/A | + | | failed to create conntrack | 0 | 0 | N/A | + | | fragment of yet incomplete | 0 | 0 | N/A | + | | packet | | | | + | | fragment out of order within | 0 | 0 | N/A | + | | host | | | | + | | fragments not supported | 0 | 0 | N/A | + | | incorrect checksum | 0 | 0 | N/A | + | | malformed IP packets | 0 | 0 | N/A | + | | packets hitting blackhole | 0 | 0 | N/A | + | | route | | | | + | | packets with unknown route | 0 | 0 | N/A | + | | packets with unknown source | 0 | 0 | N/A | + | | packets with unsupported IP | 0 | 0 | N/A | + | | options | | | | + | | too short packets | 0 | 0 | N/A | + | Other | packets hitting NAT source | 0 | 0 | N/A | + | | collision | | | | + | Redirect | neigh | 0 | 0 | N/A | + | | peer | 0 | 0 | N/A | + | | plain | 20 | 0 | N/A | + | Total | packets | 34 | 22 | N/A | + +----------+--------------------------------+---------+--------+-----+ + ``` + +## eBPF program debug logs + +Sometimes it is necessary to examine the logs that are emitted by the eBPF +programs themselves. Although the logs can be very verbose (because the +programs will log every packet), they can be invaluable to diagnose eBPF program +issues. To enable the log, set the `bpfLogLevel` Felix configuration setting to +`Debug`. + +:::caution + +Enabling logs in this way has a significant impact on eBPF program performance. + +::: + +To reduce the performance impact in production clusters, you can target logging +to specific traffic and/or specific interfaces using the +[bpfLogFilters](../../reference/component-resources/node/felix/configuration.mdx) Felix +configuration setting. Filters are pcap expressions. + +> Note that the filters are applied to the original packet, before any NAT or +encapsulation. Therefore, to log a packet that is being sent to a service making +its way via different devices, you must filter on the service IP and port and +also the backend pod IP and port. + +The logs are emitted to the kernel trace buffer, and they can be examined using the following command: + +``` +kubectl exec -n calico-system -- bpftool prog tracelog +``` + +Logs have the following format: + +``` + <...>-84582 [000] .Ns1 6851.690474: 0: ens192---E: Final result=ALLOW (-1). Program execution time: 7366ns +``` + +The parts of the log are explained below: + +- `<...>-84582` gives an indication about what program (or kernel process) was handling the + packet. For packets that are being sent, this is usually the name and PID of the program that is actually sending + the packet. For packets that are received, it is typically a kernel process, or an unrelated program that happens to + trigger the processing. +- `6851.690474` is the log timestamp. + +- `ens192---E` is the $[prodname] log tag. For programs attached to interfaces, the first part contains the + first few characters of the interface name. The suffix is either `-I` or `-E` indicating "Ingress" or "Egress". + "Ingress" and "Egress" have the same meaning as for policy: + + - A workload ingress program is executed on the path from the host network namespace to the workload. + - A workload egress program is executed on the workload to host path. + - A host endpoint ingress program is executed on the path from external node to the host. + - A host endpoint egress program is executed on the path from host to external host. + +- you may also see `ens192---X` which indicates an XDP program. $[prodname] uses XDP programs to implement `doNotTrack` policies on host devices only. + +- `Final result=ALLOW (-1). Program execution time: 7366ns` is the message. In this case, logging the final result of + the program. Note that the timestamp is massively distorted by the time spent logging. + +## Debugging policy issues + +If you suspect that $[prodname] is dropping packets due to policy, you can use the +`calico-node -bpf` tool to dump the policy that is attached to a specific interface. + +``` +kubectl exec -n calico-system -- calico-node -bpf policy dump [--asm] +``` + +Where: + - `` is the name of the interface, for example `eth0` or `caliXXXXXX`. + - `` represents location of the policy, either `ingress`, `egress`, `xdp` or `all`. + + +Dump of an ingress policy. Note that ingress policy for a pod is attached to the +`tc/tcx` egress hook of the host-side of the caliX veth pair, while ingress +policy for host endpoints is attached to the `tc/tcx` ingress hook of the host +interface. Similarly for egress policy. + +``` +IfaceName: calic31b4f7fc58 +Hook: tc egress +Error: +Policy Info: +// Start of tier default +// Start of policy default/knp.default.allow-nginx-from-ubuntu +// Start of rule action:"allow" protocol:{name:"tcp"} dst_ports:{first:80 last:80} src_ip_set_ids:"s:nzE8vwTu69FSscx2FDKjb20D9dZxEyVxsWFqwA" original_src_selector:"projectcalico.org/orchestrator == 'k8s' && app == 'ubuntu-client'" rule_id:"29vMYcPWr7reSxxN" +// IPSets src_ip_set_ids:<0x303904ae5eae5418> +// count = 9 +// End of rule 29vMYcPWr7reSxxN +// End of policy default/knp.default.allow-nginx-from-ubuntu +// End of tier default: deny +// Start of rule action:"allow" rule_id:"aBMQCbsUMESPKGRp" +// count = 0 +// End of rule aBMQCbsUMESPKGRp +``` + +Rules that use selectors refer to IP sets. You can dump the contents of an IP set using the `ipsets` command and you can check whether the IP set contains the expected members: + +``` +kubectl exec -n calico-system -- calico-node -bpf ipsets dump +``` + +You can see how many packets have matched each rule. In this example, 9 packets have matched rule `29vMYcPWr7reSxxN` and 0 packets have matched rule `aBMQCbsUMESPKGRp`. + +Adding `--asm` will show the eBPF assembly code for the program as well. + +``` +kubectl exec -n calico-system -- calico-node -bpf ipsets dump + +IP set 0x303904ae5eae5418 + 192.168.58.5/32 + +IP set 0xffef9f925a8a4ca4 + 192.168.129.66/32 + 192.168.129.68/32 +``` + +## Debugging calico-node not ready + +If you notice that a `calico-node` pod is not ready, check its logs for errors. The most likely reason for `calico-node` not being ready in eBPF mode is +that $[prodname] is not able to update a program attached to an interface. Look for the following type of warning: + +``` +2025-09-22 22:39:59.801 [WARNING][10374] felix/bpf_ep_mgr.go 2107: Failed to apply policy to endpoint, leaving it dirty +``` + +One reason for this type of error is that the eBPF programs provided in the $[nodecontainer] image are not compatible with the verifier used by your kernel. Each kernel must ensure that the eBPF programs it loads are safe to run. However, +capabilities of the verifier differ between kernel versions. We do test the eBPF programs with a range of kernels, but it is impossible to test all kernels. You may see errors such as if the verifier rejects a program: + +``` +265: (79) r1 = *(u64 *)(r10 -72) ; R1_w=ctx(off=0,imm=0) R10=fp0 +266: (79) r2 = *(u64 *)(r10 -128) ; R2_w=scalar(umin=14,umax=74,var_off=(0x2; 0x7c)) R10=fp0 +267: (b7) r5 = 0 ; R5_w=0 +268: (85) call bpf_skb_store_bytes#9 +invalid access to map value, value_size=1512 off=8 size=0 +R3 min value is outside of the allowed memory range +processed 1102 insns (limit 1000000) max_states_per_insn 2 total_states 38 peak_states 38 mark_read 27 +-- END PROG LOAD LOG -- +libbpf: prog 'calico_tc_skb_ipv4_frag': failed to load: -13 +libbpf: failed to load object '/usr/lib/calico/bpf/to_wep_no_log.o' +2025-07-31 17:36:15.708 [WARNING][45] felix/bpf_ep_mgr.go 2124: Failed to apply policy to endpoint, leaving it dirty error=attaching program to wep: loading generic v4 tc hook program: error loading program: error loading object permission denied +attaching program to wep: loading generic v4 tc hook program: error loading program: error loading object permission denied name="enif327a56b833" wepID=&types.WorkloadEndpointID{OrchestratorId:"k8s", WorkloadId:"calico-system/csi-node-driver-tggmr", EndpointId:"eth0"} +2025-07-31 17:36:15.708 [WARNING][45] felix/bpf_ep_mgr.go 2124: Failed to apply policy to endpoint, leaving it dirty error=attaching program to wep: loading generic v4 tc hook program: error loading program: error loading object permission denied +``` + +If you see errors of this type, please open an issue on the $[prodname] GitHub repository, including details of your kernel version and distribution. + +## Poor performance + +A number of problems can reduce the performance of the eBPF data plane. + +- Verify that you are using the best networking mode for your cluster. If possible, avoid using an overlay network; + a routed network with no overlay is considerably faster. If you must use one of $[prodname]'s overlay modes, + use VXLAN, not IPIP. IPIP performs poorly in eBPF mode due to kernel limitations. +- If you are not using an overlay, verify that the [Felix configuration parameters](../../reference/component-resources/node/felix/configuration.mdx) + `ipInIpEnabled` and `vxlanEnabled` are set to `false`. Those parameters control whether Felix configured itself to + allow IPIP or VXLAN, even if you have no IP pools that use an overlay. The parameters also disable certain eBPF + mode optimisations for compatibility with IPIP and VXLAN. + + To examine the configuration: + + ```bash + kubectl get felixconfiguration -o yaml + ``` + + ```yaml noValidation + apiVersion: projectcalico.org/v3 + items: + - apiVersion: projectcalico.org/v3 + kind: FelixConfiguration + metadata: + creationTimestamp: "2020-10-05T13:41:20Z" + name: default + resourceVersion: "767873" + uid: 8df8d751-7449-4b19-a4f9-e33a3d6ccbc0 + spec: + ... + ipipEnabled: false + ... + vxlanEnabled: false + kind: FelixConfigurationList + metadata: + resourceVersion: "803999" + ``` + +- If you are running your cluster in a cloud such as AWS, then your cloud provider may limit the bandwidth between + nodes in your cluster. For example, most AWS nodes are limited to 5GBit per connection. + +### Runtime profiling + +Setting `bpfProfiling` to `Enabled` enables collection of runtime profiling data +for eBPF programs. It collects the average execution time and number of +executions for each eBPF program attached to each interface. The +profiling data can be examined using the `calico-node -bpf profiling +e2e` command. The command resets the profiling data after dumping it. + +``` +----------------+-------------+-----+-------------+-------+-------------+-------+-------------+-------+ +| IFACE | INGRESS NEW | # | INGRESS EST | # | EGRESS NEW | # | EGRESS ETS | # | ++----------------+-------------+-----+-------------+-------+-------------+-------+-------------+-------+ +| lo | --- | --- | --- | --- | 142.263 ns | 10272 | --- | --- | +| eth0 | 2492.344 ns | 32 | 1535.443 ns | 16114 | 6296.421 ns | 749 | 1503.339 ns | 10982 | +| eni76136be4c77 | 5031.436 ns | 149 | 1194.923 ns | 1421 | 4950.196 ns | 138 | 1437.015 ns | 1432 | +| eni80d5c04bc95 | 7773.459 ns | 74 | 1508.973 ns | 641 | 4907.333 ns | 69 | 1715.848 ns | 646 | +| eth1 | 136.250 ns | 24 | --- | --- | 75.320 ns | 25 | --- | --- | +| eni5f8ab1cfc29 | 107.250 ns | 36 | 1068.596 ns | 1514 | 189.528 ns | 36 | 1104.335 ns | 1658 | +| bpfout.cali | 440.000 ns | 1 | --- | --- | 206.000 ns | 1 | --- | --- | ++----------------+-------------+-----+-------------+-------+-------------+-------+-------------+-------+ + +``` + +## Debug high CPU usage + +If you notice `$[noderunning]` using high CPU: + +- Check if `kube-proxy` is still running. If `kube-proxy` is still running, you must either disable `kube-proxy` or + ensure that the Felix configuration setting `bpfKubeProxyIptablesCleanupEnabled` is set to `false`. If the setting + is set to `true` (its default), then Felix will attempt to remove `kube-proxy`'s iptables rules. If `kube-proxy` is + still running, Felix will continually try to remove the rules, which can cause high CPU usage. +- If your cluster is very large, or your workload involves significant service churn, you can increase the interval + at which Felix updates the services data plane by increasing the `bpfKubeProxyMinSyncPeriod` setting. The default is + 1 second. Increasing the value has the trade-off that service updates will happen more slowly. +- $[prodname] supports endpoint slices, similarly to `kube-proxy`. If your Kubernetes cluster supports endpoint + slices and they are enabled, then you can enable endpoint slice support in $[prodname] with the + `bpfKubeProxyEndpointSlicesEnabled` configuration flag. + diff --git a/calico-cloud_versioned_docs/version-24-1/operations/ebpf/use-cases-ebpf.mdx b/calico-cloud_versioned_docs/version-24-1/operations/ebpf/use-cases-ebpf.mdx new file mode 100644 index 0000000000..628c264f63 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/ebpf/use-cases-ebpf.mdx @@ -0,0 +1,95 @@ +--- +description: Guidance on when the Calico Cloud eBPF data plane on a connected cluster fits a workload compared to the standard iptables data plane, with trade-offs. +--- + +# eBPF use cases + +## Big picture + +Learn when to use eBPF (and when not to). + +## What is eBPF? + +eBPF is a feature available in Linux kernels that allows you to run a virtual machine inside the kernel. This virtual machine allows you to safely load programs into the kernel, to customize its operation. Why is this important? + +In the past, making changes to the kernel was difficult: there were APIs you could call to get data, but you couldn’t influence what was inside the kernel or execute code. Instead, you had to submit a patch to the Linux community and wait for it to be approved. With eBPF, you can load a program into the kernel and instruct the kernel to execute your program if, for example, a certain packet is seen or another event occurs. + +With eBPF, the kernel and its behavior become highly customizable, instead of being fixed. This can be extremely beneficial, when used under the right circumstances. + +## $[prodname] and eBPF + +$[prodname] offers an eBPF data plane as an alternative to our standard Linux data plane (which is iptables based). While the standard data plane focuses on compatibility by working together with kube-proxy and your own iptables rules, the eBPF data plane focuses on performance, latency, and improving user experience with features that aren’t possible with the standard data plane. + +But $[prodname] doesn’t only support standard Linux and eBPF; it currently supports a total of three data planes, including Windows HNS, and has plans to add support for even more data planes in the near future. $[prodname] enables you, the user, to decide what works best for what you want to do. + +If you enable eBPF within $[prodname] but have existing iptables flows, we won’t touch them. Because maybe you want to use connect-time load balancing, but leave iptables as is. With $[prodname], it’s not an all-or-nothing deal—we allow you to easily load and unload our eBPF data plane to suit your needs, which means you can quickly try it out before making a decision. $[prodname] offers you the ability to leverage eBPF as needed, as an additional control to build your Kubernetes cluster security. + +## Use cases + +There are several use cases for eBPF, including traffic control, creating network policy, and connect-time load balancing. + +### Traffic control + +Without eBPF, packets use the standard Linux networking path on their way to a final destination. If a packet shows up at point A, and you know that the packet needs to go to point B, you can optimize the network path in the Linux kernel by sending it straight to point B. With eBPF, you can leverage additional context to make these changes in the kernel so that packets bypass complex routing and simply arrive at their final destination. + +This is especially relevant in a Kubernetes container environment, where you have numerous networks. (In addition to the host network stack, each container has its own mini network stack.) When traffic comes in, it is usually routed to a container stack and must travel a complex path as it makes its way there from the host stack. This routing can be bypassed using eBPF. + +### Creating network policy + +When creating network policy, there are two instances where eBPF can be used: + +- **eXpress Data Path (XDP)** – As a raw packet buffer enters the system, eBPF gives you an efficient way to examine that buffer and make quick decisions about what to do with it. + +- **Network policy** – eBPF allows you to efficiently examine a packet and apply network policy, both for pods and hosts. + +### Connect-time load balancing + +When load balancing service connections in Kubernetes, a port needs to talk to a service and therefore network address translation (NAT) must occur. A packet is sent to a virtual IP, and that virtual IP translates it to the destination IP of the pod backing the service; the pod then responds to the virtual IP and the return packet is translated back to the source. + +With eBPF, you can avoid this packet translation by using an eBPF program that you’ve loaded into the kernel and load balancing at the source of the connection. All NAT overhead from service connections is removed because destination network address translation (DNAT) does not need to take place on the packet processing path. + +## The price of performance + +So is eBPF more efficient than standard Linux iptables? The short answer: it depends. + +If you were to micro-benchmark how iptables works when applying network policies with a large number of IP addresses (i.e. ipsets), iptables in many cases is better than eBPF. But if you want to do something in the Linux kernel where you need to alter the packet flow in the kernel, eBPF would be the better choice. Standard Linux iptables is a complex system and certainly has its limitations, but at the same time it provides options to manipulate traffic; if you know how to program iptables rules, you can achieve a lot. eBPF allows you to load your own programs into the kernel to influence behavior that can be customized to your needs, so it is more flexible than iptables as it is not limited to one set of rules. + +Something else to consider is that, while eBPF allows you to run a program, add logic, redirect flows, and bypass processing—which is a definite win—it’s a virtual machine and as such must be translated to bytecode. By comparison, the Linux kernel’s iptables is already compiled to code. + +As you can see, comparing eBPF to iptables is not a straight apples-to-apples comparison. What we need to assess is performance, and the two key factors to look at here are latency (speed) and expense. If eBPF is very fast but takes up 80% of your resources, then it’s like a Lamborghini—an expensive, fast car. And if that works for you, great (maybe you really like expensive, fast cars). Just keep in mind that more CPU usage means more money spent with your cloud providers. So while a Lamborghini might be faster than a lot of other cars, it might not be the best use of money if you need to comply with speed limits on your daily commute. + +## When to use eBPF (and when not to) + +With eBPF, you get performance—but it comes at a cost. You need to find a balance between the two by figuring out the price of performance, and deciding if it’s acceptable to you from an eBPF perspective. + +Let’s look at some specific cases where it would make sense to use eBPF, and some where it would not. + +### When not to use eBPF + +### ✘ Packet-by-packet processing + +Using eBPF to perform CPU intensive or packet-by-packet processing, such as decryption and re-encryption for encrypted flows, would not be efficient because you would need to build a structure and do a lookup for every packet, which is expensive. + +### When to use eBPF + +### ✔ XDP + +eBPF provides an efficient way to examine raw packet buffers as they enter the system, allowing you to make quick decisions about what to do with them. + +### ✔ Connect-time load balancing + +With eBPF, you can load balance at the source using a program you’ve loaded into the kernel, instead of using a virtual IP. Since DNAT does not need to take place on the packet processing path, all NAT overhead from service connections is removed. + +### ✔ Building a service mesh control plane + +Service mesh relies on proxies like Envoy. A lot of thought has gone into designing this process over the years. The main reason for doing it this way is that, in many cases, it is not viable to do inline processing for application protocols like HTTP at the high speeds seen inside a cluster. Therefore, you should think of using eBPF to route traffic to a proxy like Envoy in an efficient way, rather than using it to replace the proxy itself. However, you do need to turn off connect-time load balancing (CTLB) so sidecars can see the service addresses. Given you are already taking a performance hit by the extra hop to the sidecar, not using CTLB performance optimization to avoid NAT overhead is likely not a big deal. + +## Summary + +Is eBPF a replacement for iptables? Not exactly. It’s hard to imagine everything working as efficiently with eBPF as it does with iptables. For now, the two co-exist and it’s up to the user to weigh the price-performance tradeoff and decide which feature to use when, given their specific needs. + +We believe the right solution is to leverage eBPF, along with existing mechanisms in the Linux kernel, to achieve your desired outcome. That’s why $[prodname] offers support for multiple data planes, including standard Linux, Windows HNS, and Linux eBPF. Since we have established that both eBPF and iptables are useful, the only logical thing to do in our opinion is to support both. $[prodname] gives you the choice so you can choose the best tool for the job. + +## Additional resources + +To learn more and see performance metrics from our test environment, see the blog, [Introducing the eBPF data plane](https://www.projectcalico.org/introducing-the-calico-ebpf-dataplane/). diff --git a/calico-cloud_versioned_docs/version-24-1/operations/index.mdx b/calico-cloud_versioned_docs/version-24-1/operations/index.mdx new file mode 100644 index 0000000000..f8da8da396 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/index.mdx @@ -0,0 +1,41 @@ +--- +description: Day-2 operations for Calico Cloud connected clusters covering cluster management, secure component communications, monitoring, eBPF, and disconnection. +--- + +import { DocCardLink, DocCardLinkLayout } from '/src/___new___/components'; + +# Operations + +Post-installation tasks for managing Calico Cloud. + + + + + + + + +## Monitoring + + + + + + + + + + + + +## eBPF + + + + + + + + + + diff --git a/calico-cloud_versioned_docs/version-24-1/operations/monitor/index.mdx b/calico-cloud_versioned_docs/version-24-1/operations/monitor/index.mdx new file mode 100644 index 0000000000..66476dc8c0 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/monitor/index.mdx @@ -0,0 +1,11 @@ +--- +description: Reference index for monitoring Calico Cloud connected clusters with Prometheus metrics covering BGP, policy, log collection, and component health. +hide_table_of_contents: true +--- + +# Monitoring + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/bgp-metrics.mdx b/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/bgp-metrics.mdx new file mode 100644 index 0000000000..74def9953e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/bgp-metrics.mdx @@ -0,0 +1,157 @@ +--- +description: Monitor BGP peering and route exchange in Calico Cloud connected clusters by defining Prometheus rules and thresholds for peer health and route counts. +--- + +# BGP metrics + +## Big picture + +Use Prometheus configured for $[prodname] `$[noderunning]` to monitor the health of BGP peers within your cluster. + +## Value + +Using the open-source Prometheus monitoring and alerting toolkit, you can view time-series metrics from $[prodname] components in the Prometheus or Grafana interfaces. + +$[prodname] adds the ability to monitor high-level operations between BGP peers in your cluster. By defining a set of simple rules and thresholds, you can monitor peer-to-peer connection health between your nodes as well as the number of routes being exchanged and receive alerts when it exceeds configured thresholds. + +## Concepts + +``` + +-------------------+ + | Host | + | +-------------------+ +------------+ +------------+ + | | Host |------------->--| | | |--->-- + | | +-------------------+ policy | Prometheus | | Prometheus | alert + +-| | Host |----------->--| Server |-->--| Alert- |--->-- + | | +-------------+ | metrics | | | manager | mechanisms + +-| | BGP Metrics |-------------->--| | | |--->-- + | | Server | | | | | | + | +-------------+ | +------------+ +------------+ + +-------------------+ ^ ^ + | | + Collect and store metrics. Web UI for accessing alert + WebUI for accessing and states. + querying metrics. Configure fan out + Configure alerting rules. notifications to different + alert receivers. +``` + +BGP metric reporting is accomplished using three key pieces: + +- BGP Metrics Server +- Prometheus Server +- Prometheus Alertmanager + +### About Prometheus + +The Prometheus scrapes various instrumented jobs (endpoints) to collect time series data for a given set of metrics. Time series data can then be queried and rules can be setup to monitor specific thresholds to trigger alerts. The data can also be visualized (such as using Grafana). + +Prometheus Server deployed as part of the $[prodname] scrapes every configured `$[noderunning]` target. Alerting rules querying BGP metrics can be configured in Prometheus and when triggered, fire alerts to the Prometheus Alertmanager. + +Prometheus Alertmanager (or simply Alertmanager), deployed as part of the $[prodname], receives alerts from Prometheus and forwards alerts to various alerting mechanisms such as _Pager Duty_, or _OpsGenie_. + +### About $[prodname] `$[noderunning]` + +`$[noderunning]` bundles together the components required for networking containers with $[prodname]. The key components are: + +- Felix +- BIRD +- confd + +Its critical function means that it runs on every machine that provides endpoints. A binary running inside `$[noderunning]` monitors the BIRD daemon for peering and routing activity and reports these statics to Prometheus. + +## How to + +BGP metrics are generated within `$[noderunning]` every 5 seconds using statistics pulled from the BIRD daemon. + +The metrics generated are: + +- `bgp_peers` - Total number of peers with a specific BGP connection status. +- `bgp_routes_imported` - Current number of routes successfully imported into the routing table. +- `bgp_route_updates_received` - Total number of route updates received over time (since startup). + +$[prodname] will run BGP metrics for Prometheus by default. Metrics are available on each compute node at `https://:9900/metrics`, secured with mTLS. + +To access BGP metrics directly, you must use the TLS credentials: + +1. Extract the TLS credentials and CA bundle from the cluster. + + ```bash + kubectl get secret -n tigera-prometheus calico-node-prometheus-client-tls -o jsonpath='{.data.tls\.key}' | base64 -d > key.pem + kubectl get secret -n tigera-prometheus calico-node-prometheus-client-tls -o jsonpath='{.data.tls\.crt}' | base64 -d > cert.pem + kubectl get cm -n tigera-prometheus tigera-ca-bundle -o jsonpath='{.data.tigera-ca-bundle\.crt}' > bundle.pem + ``` + +1. Verify you can access the metrics. + + ```bash + curl --cacert bundle.pem --key key.pem --cert cert.pem https://:9900/metrics + ``` + +Refer to [Configuring Prometheus](../prometheus/index.mdx) for information on how to create a new Alerting rule or updating the scraping interval for how often Prometheus collects the metrics. + +### BGP peers metric + +The metric `bgp_peers` has the relevant labels `instance`, `status` and `ip_version`. Using this metric, you can identify how many peers have a specific BGP connection status with a given node instance and IP version. This metric will be available as a combination of `{instance, status, ip_version}`. + +Example queries: + +- Total number of peers currently with a BGP connection to the node instance “calico-node-1”, with status “Established”, for IP version “IPv4”. + +``` +bgp_peers{instance="calico-node-1", status="Established", ip_version="IPv4"} +``` + +- Total number of peers currently with a BGP connection to the node instance “calico-node-1”, with status “Down”, for IP version “IPv6”. + +``` +bgp_peers{instance="calico-node-1", status="Down", ip_version="IPv6"} +``` + +- Total number of peers currently with a BGP connection to any node instance, with a status that is not “Established”, for IP version “IPv4”. + +``` +bgp_peers{status!="Established", ip_version="IPv4"} +``` + +Valid BGP connection statuses are: "Idle", "Connect", "Active", "OpenSent", "OpenConfirm", "Established", "Close", "Down" and "Passive". + +### BGP routes imported metric + +The metric `bgp_routes_imported` has the relevant labels `instance` and `ip_version`. Using this metric, you can identify how many routes are being successfully imported into a given node instance's routing table at a specific point in time. This number can increase or decrease depending on how BGP rules process incoming routes. This metric will be available as a combination of `{instance, ip_version}`. + +Example queries: + +- Computes the per-second rate for the number of routes imported by a specific node instance “calico-node-1” looking up to 120 seconds back (using the two most recent data points). + +``` +irate(bgp_routes_imported{instance="calico-node-1",ip_version="IPv4"}[120s]) +``` + +- Computes the per-second rate for the number of routes imported across all node instances looking up to 120 seconds back (using the two most recent data points). + +``` +irate(bgp_routes_imported{ip_version="IPv4"}[120s]) +``` + +### BGP route updates received metric + +The metric `bgp_route_updates_received` has the relevant labels `instance` and `ip_version`. Using this metric, you can identify the total number of BGP routes received by a given node over time. This number includes all routes that have been accepted & imported into the routing table, as well as any routes that were rejected as invalid, rejected by filters or rejected as already in the route table. This total number should only increase over time. This metric will be available as a combination of `{instance, ip_version}`. + +Example queries: + +- Computes the per-second rate for the number of routes received by a specific node instance “calico-node-1” looking up to 5 minutes back (using the two most recent data points). + +``` +irate(bgp_route_updates_received{instance="calico-node-1",ip_version="IPv4"}[5m]) +``` + +- Computes the per-second rate for the number of routes received across all node instances looking up to 5 minutes back (using the two most recent data points). + +``` +irate(bgp_route_updates_received{ip_version="IPv4"}[5m]) +``` + +## Additional resources + +- [Configuring Prometheus](../prometheus/index.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/elasticsearch-and-fluentd-metrics.mdx b/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/elasticsearch-and-fluentd-metrics.mdx new file mode 100644 index 0000000000..842f83e06b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/elasticsearch-and-fluentd-metrics.mdx @@ -0,0 +1,55 @@ +--- +description: Track Calico Cloud Fluentd metrics in Prometheus to alert on flow, DNS, and audit log collection disruptions on connected clusters. +--- + +# Fluentd metrics + +## Big picture + +Use the Prometheus monitoring and alerting tool for Fluentd metrics to ensure continuous network visibility. + +## Value + +Platform engineering teams rely on logs for visibility into their networks. If collecting or storing logs are disrupted, this can impact network visibility. Prometheus can monitor log collection and storage metrics so platform engineering teams are alerted about problems before they occur. + +## Concepts + +| Component | Description | +| ---------- | ------------------------------------------------------------ | +| Prometheus | Monitoring tool that scrapes metrics from instrumented jobs and displays time series data in a visualizer (such as Grafana). For $[prodname], the “jobs” that Prometheus can harvest metrics from the Fluentd component. | +| Fluentd | Sends $[prodname] logs to Elasticsearch for storage. | + +## How to + +### Create Prometheus alerts for Fluentd + +The following example creates a Prometheus rule to monitor some important Fluentd metrics, and alert when they have crossed certain thresholds: + +```yaml noValidation +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: tigera-prometheus-log-collection-monitoring + namespace: tigera-prometheus + labels: + role: tigera-prometheus-rules + prometheus: calico-node-prometheus +spec: + groups: + - name: tigera-log-collection.rules + rules: + - alert: FluentdPodConsistentlyLowBufferSpace + expr: avg_over_time(fluentd_output_status_buffer_available_space_ratio[5m]) < 75 + labels: + severity: Warning + annotations: + summary: "Fluentd pod {{$labels.pod}}'s buffer space is consistently below 75 percent capacity." + description: "Fluentd pod {{$labels.pod}} has very low buffer space. There may be connection issues between Elasticsearch +and Fluentd or there are too many logs to write out, check the logs for the Fluentd pod." +``` + +#### The alerts created in the example are described as follows: + +| Alert | Severity | Requires | Issue/reason | +| ---------------------------------------- | --------------------- | -------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| **FluentdPodConsistentlyLowBufferSpace** | Non-critical, warning | Immediate investigation to ensure logs are being gathered correctly. | A Fluentd pod’s available buffer size has averaged less than 75% over the last 5 minutes.

This could mean Fluentd is having trouble communicating with the Elasticsearch cluster, the Elasticsearch cluster is down, or there are simply too many logs to process. | diff --git a/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/index.mdx b/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/index.mdx new file mode 100644 index 0000000000..af12857877 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/index.mdx @@ -0,0 +1,11 @@ +--- +description: Reference index for Calico Cloud Prometheus metrics on connected clusters covering BGP, policy, log collection, and recommended dashboards. +hide_table_of_contents: true +--- + +# Metrics + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/operator-metrics.mdx b/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/operator-metrics.mdx new file mode 100644 index 0000000000..43795bf49e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/operator-metrics.mdx @@ -0,0 +1,113 @@ +--- +description: Monitor the Calico Cloud operator on connected clusters with Prometheus metrics for component health and TLS certificate expiry. +--- + +# Operator metrics + +## Big picture + +Use Prometheus to monitor the $[prodname] operator. + +## Value + +The $[prodname] operator exposes Prometheus metrics that give you visibility into the overall health of your $[prodname] installation. +These metrics let you set up alerts for degraded components and expiring TLS certificates before they impact operations. + +## Before you begin + +Operator metrics are enabled by default. The `tigera-operator` deployment ships with the following environment variables already set: + +- `METRICS_ENABLED=true` +- `METRICS_SCHEME=https` + +Metrics are served on port **9484** over HTTPS and scraped by the built-in Prometheus instance via the `tigera-operator-metrics` ServiceMonitor. + +## Metrics reference + +### Component status + +The `tigera_operator_component_status` metric reports the status of each $[prodname] component as managed by the operator. This mirrors the information available through `kubectl get tigerastatus`. + +| Metric | Labels | Description | +|---|---|---| +| `tigera_operator_component_status` | `component`, `condition` | Status of a component. Value is `1` (true) or `0` (false). | + +**Labels:** + +- `component` — The $[prodname] component (e.g. `calico`, `apiserver`, `monitor`, `log-collector`) +- `condition` — One of: + - `available` — The component is running and healthy + - `degraded` — The component is in an error state + - `progressing` — The component is being updated or is starting up + +**Example queries:** + +- Find all degraded components: + + ``` + tigera_operator_component_status{condition="degraded"} == 1 + ``` + +- Check if a specific component is available: + + ``` + tigera_operator_component_status{component="calico", condition="available"} + ``` + +### TLS certificate expiry + +The `tigera_operator_tls_certificate_expiry_timestamp_seconds` metric reports the expiry time of each TLS certificate managed by the operator. + +| Metric | Labels | Description | +|---|---|---| +| `tigera_operator_tls_certificate_expiry_timestamp_seconds` | `name`, `namespace`, `issuer` | Unix timestamp when the certificate expires. | + +**Labels:** + +- `name` — The Secret name containing the certificate +- `namespace` — The namespace of the Secret +- `issuer` — The certificate issuer (e.g. `tigera-operator-signer`) + +**Example queries:** + +- Certificates expiring within 30 days: + + ``` + tigera_operator_tls_certificate_expiry_timestamp_seconds - time() < 30 * 24 * 3600 + ``` + +- Certificates expiring within 7 days: + + ``` + tigera_operator_tls_certificate_expiry_timestamp_seconds - time() < 7 * 24 * 3600 + ``` + +## Built-in alerts + +$[prodname] installs a PrometheusRule resource named `calico` with alerting rules that use these metrics. You can view it with: + +```bash +kubectl -n tigera-prometheus get prometheusrule calico -o yaml +``` + +The built-in rules include: + +| Alert | Condition | Severity | +|---|---|---| +| `DeniedPacketsRate` | Denied packets rate > 50/s | info | +| `TLSCertExpiringWarning` | Certificate expires in < 30 days | warning | +| `TLSCertExpiringCritical` | Certificate expires in < 7 days | critical | +| `ComponentDegradedWarning` | Component degraded for > 15m | warning | +| `ComponentDegradedCritical` | Component degraded for > 30m | critical | +| `ComponentProgressingWarning` | Component progressing for > 15m | warning | +| `ComponentProgressingCritical` | Component progressing for > 30m | critical | + +The rule also includes two license expiry alerts, `LicenseExpiringWarning` and `LicenseExpiringCritical`. +Tigera manages licensing for $[prodname], so you don't need to act on them. + +To route these alerts, see [Configure Alertmanager](../prometheus/alertmanager.mdx). + +## Additional resources + +- [Configure Prometheus](../prometheus/configure-prometheus.mdx) +- [BYO Prometheus](../prometheus/byo-prometheus.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/policy-metrics.mdx b/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/policy-metrics.mdx new file mode 100644 index 0000000000..fc49d366a1 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/policy-metrics.mdx @@ -0,0 +1,142 @@ +--- +description: Monitor the runtime effect of policies on Calico Cloud connected clusters by defining Prometheus rules and thresholds that fire alerts on policy hits. +--- + +# Policy metrics + +$[prodname] adds the ability to monitor effects of policies configured in your cluster. +By defining a set of simple rules and thresholds, you can monitor traffic metrics and receive +alerts when it exceeds configured thresholds. + +``` + +------------+ + | | + | TSEE | + | Manager | + | | + | | + | | + +------------+ + ^ + | + | + | + +-----------------+ | + | Host | | + | +-----------------+ +------------+ +------------+ + | | Host |------------->--| | | |--->-- + | | +-----------------+ policy | Prometheus | | Prometheus | alert + +-| | Host |----------->--| Server |-->--| Alert |--->-- + | | +----------+ | metrics | | | Manager | mechanisms + +-| | Felix |-------------->--| | | |--->-- + | +----------+ | +------------+ +------------+ + +-----------------+ ^ ^ + | | + Collect and store metrics. Web UI for accessing alert + WebUI for accessing and states. + querying metrics. Configure fan out + Configure alerting rules. notifications to different + alert receivers. +``` + +Policy inspection and reporting is accomplished using four key pieces: + +- A $[prodname] specific Felix binary running inside `$[noderunning]` container + monitors the host for denied/allowed packets and collects metrics. +- Prometheus Server(s) deployed as part of the $[prodname] manifest scrapes + every configured `$[noderunning]` target. Alerting rules querying denied packet + metrics are configured in Prometheus and when triggered, fire alerts to + the Prometheus Alertmanager. +- Prometheus Alertmanager (or simply Alertmanager), deployed as part of + the $[prodname] manifest, receives alerts from Prometheus and forwards + alerts to various alerting mechanisms such as _Pager Duty_, or _OpsGenie_. +- the $[prodname] web console, also deployed as part of the $[prodname] manifest, + processes the metrics using pre-defined Prometheus queries and provides dashboards and associated workflows. + +Metrics will only be generated at a node when there are packets directed at an endpoint that are being actively profiled by a policy. +Once generated they stay alive for 60 seconds. + +Once Prometheus scrapes a node and collects policy metrics, it will be +available at Prometheus until the metric is considered _stale_, i.e., +Prometheus has not seen any updates to this metric for some time. This time is +configurable. Refer to +[Configuring Prometheus configuration](../prometheus/index.mdx) +for more information. + +Because of metrics being expired, as just described, it is entirely possible +for a GET on a metrics query URL to return no information. This is expected +if there have not been any packets being processed by a policy on that node, in +the last 60 seconds. + +Metrics generated by each $[prodname] node are: + +- `calico_denied_packets` - Total number of packets denied by $[prodname] policies. +- `calico_denied_bytes` - Total number of bytes denied by $[prodname] policies. +- `cnx_policy_rule_packets` - Sum of allowed/denied packets over rules processed by + $[prodname] policies. +- `cnx_policy_rule_bytes` - Sum of allowed/denied bytes over rules processed by + $[prodname] policies. +- `cnx_policy_rule_connections` - Sum of connections over rules processed by $[prodname] + policies. + +The metrics `calico_denied_packets` and `calico_denied_bytes` have the labels `policy` and `srcIP`. +Using these two metrics, one can identify the policy that denied packets as well as +the source IP address of the packets that were denied by this policy. Using +Prometheus terminology, `calico_denied_packets` is the metric name and `policy` +and `srcIP` are labels. Each one of these metrics will be available as a +combination of `{policy, srcIP}`. + +Example queries: + +- Total number of bytes, denied by $[prodname] policies, originating from the IP address "10.245.13.133" + by `k8s_ns.ns-0` profile. + +``` +calico_denied_bytes{policy="profile|k8s_ns.ns-0|0|deny", srcIP="10.245.13.133"} +``` + +- Total number of packets denied by $[prodname] policies, originating from the IP address "10.245.13.149" + by `k8s_ns.ns-0` profile. + +``` +calico_denied_packets{policy="profile|k8s_ns.ns-0|0|deny", srcIP="10.245.13.149"}} +``` + +The metrics `cnx_policy_rule_packets`, `cnx_policy_rule_bytes` and `cnx_policy_rule_connections` have the +labels: `tier`, `policy`, `namespace`, `rule_index`, `action`, `traffic_direction`, `rule_direction`. + +Using these metrics, one can identify allow, and denied byte rate and packet rate, both inbound and outbound, indexed by both policy and rule. the $[prodname] web console Dashboard makes heavy usage of these metrics. +Staged policy names are prefixed with "staged:". + +Example queries: + +- Query counts for rules: Packet rates for specific rule by traffic_direction + +``` +sum(irate(cnx_policy_rule_packets{namespace="namespace-2",policy="policy-0",rule_direction="ingress",rule_index="rule-5",tier="tier-0"}[30s])) without (instance) +``` + +- Query counts for rules: Packet rates for each rule in a policy by traffic_direction + +``` +sum(irate(cnx_policy_rule_packets{namespace="namespace-2",policy="policy-0",tier="tier-0"}[30s])) without (instance) +``` + +- Query counts for a single policy by traffic_direction and action + +``` +sum(irate(cnx_policy_rule_packets{namespace="namespace-2",policy="policy-0",tier="tier-0"}[30s])) without (instance,rule_index,rule_direction) +``` + +- Query counts for all policies across all tiers by traffic_direction and action + +``` +sum(irate(cnx_policy_rule_packets[30s])) without (instance,rule_index,rule_direction) +``` + +See the +[Felix configuration reference](../../../reference/component-resources/node/felix/configuration.mdx) for +the settings that control the reporting of these metrics. $[prodname] manifests +normally set `PrometheusReporterEnabled=true` and +`PrometheusReporterPort=9081`, so these metrics are available on each compute +node at `http://:9081/metrics`. diff --git a/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/recommended-metrics.mdx b/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/recommended-metrics.mdx new file mode 100644 index 0000000000..e793fbf252 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/monitor/metrics/recommended-metrics.mdx @@ -0,0 +1,562 @@ +--- +description: Recommended Prometheus metrics for Calico Cloud Typha, Felix, and policy components on connected clusters, covering the signals most critical to cluster health. +--- + +# Recommended Prometheus metrics + +## Big picture + +Monitor the $[prodname] Typha, Felix, and policy component metrics to ensure optimal cluster operation. + +## Concepts + +$[prodname] Typha, Felix, and policy components are the most critical to monitor because they are responsible for ensuring networking and security functions are up-to-date and working as expected. + +### Typha + +Typha is a caching datastore proxy that sits between calico-nodes and Kubernetes API Server. Its primary function is to allow for increased cluster scale by reducing the load on Kubernetes API Server. Without Typha, large clusters (200+ nodes) would need a considerable amount of memory to correspond to the continuous watches and requests from calico-nodes running in the cluster. + +Typha maintains a single datastore connection on behalf of all of its clients (processes running in the calico-node pods, with Felix being Typhas’ main client). Typha watches for node, pod, network policy, bgp configuration, and other events on the Kubernetes API Server, caches and deduplicates this data, and fans out these events to its clients. + +### Felix + +Felix is a component of calico-node and is responsible for $[prodname] network policy. +Felix must be continuously in sync with the datastore to ensure the correct set of policies are applied to the node it is running on. + +![Typha-felix](/img/calico-enterprise/typha-felix.png) + +### About metrics + +Each $[prodname] component that you want to connect to Prometheus for endpoint metrics has its own configuration (bgp, license, policy, felix, and typha). + +Note that Felix is a separate application with metric endpoints, its own core metrics to monitor itself, and a separate port for a second policy metric endpoint. + +## Metrics + +This section provides metrics recommendations for maintaining optimal cluster operations. Note the following: + +- Threshold values for each metric depend on the cluster size and churn rate. +- Threshold recommendations are provided where possible, but because each cluster is different, and metrics can depend on cluster churn rate and scale. We recommend that you baseline the cluster to establish numbers that represent normal figures for your cluster. +- Metrics that start increasing rapidly from the baseline set need attention. + +Typha +- [Typha general metrics](#typha-general-metrics) +- [Typha cluster mesh metrics](#typha-cluster-mesh-metrics) +- [Typha client metrics](#typha-client-metrics) +- [Typha cache internals](#typha-cache-internals) +- [Typha snapshot details](#typha-snapshot-details) + +Felix +- [Policy metrics](#policy-metrics) +- [Felix cluster state metrics](#felix-cluster-state-metrics) +- [Felix error metrics](#felix-error-metrics) +- [Felix time-based metrics](#felix-time-based-metrics) + +## Typha general metrics + +### Datastore cache size + +| Datastore cache size | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | Note: Syncer (type) is Typha's internal name for a client (type).
Individual syncer values:
(typha_cache_size\{syncer="bgp"\})
(typha_cache_size\{syncer="dpi"\})
(typha_cache_size\{syncer="felix"\})
(typha_cache_size\{syncer="node-status"\})
(typha_cache_size\{syncer="tunnel-ip-allocation"\})

Sum of all syncers:
The sum of all cache sizes (each syncer type has a cache).
sum by (instance) (typha_cache_size)

Largest syncer:
max by (instance) (typha_cache_size) | +| Example value | Example of: max by (instance) (typha_cache_size\{syncer="felix"\})

\{instance="10.0.1.20:9093"\} 661
\{instance="10.0.1.31:9093"\} 661 | +| Explanation | The total number of key/value pairs in Typha's in-memory cache.This metric represents the scale of the $[prodname] datastore as it tracks how many WEPs (pods and services), HEPs (hostendpoints), networksets, globalnetworksets, $[prodname] Network Policies etc that Typha is aware of across the entire Calico Federation.You can use this metric to monitor individual syncers to Typha (like Felix, BGP etc), or to get a sum of all syncers. We recommend that you monitor the largest syncer but it is completely up to you. This is a good metric to understand how much data is in Typha. Note: If all Typhas are in sync then they should have the same value for this metric. | +| Threshold value recommendation | The value of this metric will depend on the scale of the Calico Federation and will always increase as WEPs, $[prodname] network policies and clusters are added. Achieve a baseline first, then monitor for any unexpected increases from the baseline. | +| Threshold breach symptoms | Unexpected increases may indicate memory leaks and performance issues with Typha. | +| Threshold breach recommendations | Check CPU usage on Typha pods and Kubernetes nodes. Increase resources if needed, rollout and restart Typha(s) if needed. | +| Priority level | Optional. | + +### CPU usage + +| CPU usage | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | rate(process_cpu_seconds_total\{30s\}) \* 100 | +| Example value | \{endpoint="metrics-port", instance="10.0.1.20:9093", job="typha-metrics-svc", namespace="calico-system", pod="calico-typha-6c6cc9fcf7-csbdl", service="typha-metrics-svc"\} 0.27999999999999403 | +| Explanation | CPU in use by Typha represented as a percentage of a core. | +| Threshold value recommendation | A spike at startup is normal. It is recommended to achieve a baseline first, then monitor for any unexpected increases from this baseline. A rule of thumb is to investigate maintained CPU usage above 90%. | +| Threshold breach symptoms | Unexpected maintained CPU usage could cause Typha to fall behind in updating its clients (for example, Felix) and could cause delays to policy updates. | +| Threshold breach recommendations | Check CPU usage on Kubernetes nodes. If needed, increase resources, and rollout restart Typha(s). | +| Priority level | Recommended. | + +### Memory usage + +| Memory usage | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | process_resident_memory_bytes | +| Example value | process_resident_memory_bytes\{endpoint="metrics-port", instance="10.0.1.20:9093", job="typha-metrics-svc", namespace="calico-system", pod="calico-typha-6c6cc9fcf7-csbdl", service="typha-metrics-svc"\} 80515072 | +| Explanation | Amount of memory used by Typha. | +| Threshold value recommendation | It is recommended to achieve a baseline first, then monitor for any unexpected increases from this baseline. A rule of thumb is to investigate if maintained memory usage is above 90% of what is available from the underlying node. The metric can also be used for memory leaks. In this case, the metric would show Typhas' memory consumption rising over time, even though the cluster is in a stable state. | +| Threshold breach symptoms | Unexpected maintained memory usage could cause Typha to fall behind in updating its clients (for example, Felix) and could cause delays to policy updates. | +| Threshold breach recommendations | Check memory usage on Kubernetes nodes. Increase resources if needed, and rollout restart Typha(s) if needed. | +| Priority level | Recommended. | + +## Typha cluster mesh metrics + +The following metrics are applicable only if you have implemented [Cluster mesh](multicluster/overview.mdx). + +Note that this metric requires a count syntax because you will have a copy of the metric per RemoteClusterConfiguration. As shown in the table, the value `2 = In Sync` reflects good connections. + +``` +remote_cluster_connection_status\{cluster="foo"\} = 2 +remote_cluster_connection_status\{cluster="bar"\} = 2 +remote_cluster_connection_status\{cluster="baz"\} = 1 +``` + +### Remote cluster connections (in-sync) + +| Remote cluster connections (in-sync) | | +| ------------------------------------ | ------------------------------------------------------------ | +| Metric | count by (instance) (remote_cluster_connection_status == 2) | +| Explanation | This represents the number of remote cluster connections that are connected and in sync. Each remote cluster will report a *connection_status* value from the following list:
- 0 = Not Connected
- 1 = Connecting
- 2 = In Sync
- 3 = Resync in Process
- 4 = Config Change Restart Required

We suggest the count syntax because there will be one copy of *remote_cluster_connection_status* per cluster: - remote_cluster_connection_status[cluster="foo"] = 2
remote_cluster_connection_status[cluster="bar"] = 2
remote_cluster_connection_status[cluster="baz"] = '

Counting the number of metrics with value 2 returns the number of In Sync clusters. | +| Threshold value recommendation | When remote cluster connections are initializing, *connection_status* values will fluctuate. After the connection is established, this value should be equal to the number of remote clusters in the environment (if everything is in sync). | +| Threshold breach symptoms | N/A
For out-of-sync symptoms, see the out-of-sync metric. | +| Threshold breach recommendations | N/A
For out-of-sync recommendations, see the out-of-sync metric. | +| Priority level | Recommended. | + +### Remote cluster connections (out-of-sync) + +The following metrics are applicable only if you have implemented [Cluster mesh](multicluster/overview.mdx). + +| Remote cluster connections (out-of-sync) | | +| ---------------------------------------- | ------------------------------------------------------------ | +| Metric | count by (instance) (remote_cluster_connection_status != 2) | +| Explanation | Number of remote cluster connections that are not in sync (i.e. resyncing or failing to connect). Each remote cluster will report a *connection_status* value from the following list:
- 0 = Not Connected
- 1 = Connecting
- 2 = In Sync
- 3 = Resync in Process
- 4 = Config Change Restart Required | +| Threshold value recommendation | This value should be 2 if everything is in sync. Note: At Typha startup, it is normal to have non-2 values, but it should stabilize at 2 after connections come up. | +| Threshold breach symptoms | Typha will not receive updates from the relevant remote clusters. Connected clients will see stale or partial data from remote clusters. | +| Threshold breach recommendations | Investigate Typha's logs where remote cluster connectivity events are logged. Ensure the networking between clusters is not experiencing issues. | +| Priority level | Recommended. | + +## Typha client metrics + +### Total connections accepted + +| Total connections accepted | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | typha_connections_accepted | +| Example value | typha_connections_accepted\{endpoint="metrics-port", instance="10.0.1.20:9093", job="typha-metrics-svc", namespace="calico-system", pod="calico-typha-6c6cc9fcf7-csbdl", service="typha-metrics-svc"\} 10 | +| Explanation | Total number of connections accepted over time. This value always increases. | +| Threshold value recommendation | A steady increase over time is normal. Counters rising after a Felix or Typha restart is also normal (as clients get rebalanced). Investigate connection counters that rise rapidly with no Felix or Typha restarts. | +| Threshold breach symptoms | Counters rising when there are no Felix or Typha restarts, or no action that could cause restarts (an upgrade for example), could indicate unexpected Felix or Typha restarts or issues. | +| Threshold breach recommendations | Check resource usage on Typha(s) and Kubernetes nodes. Increase resources if needed. | +| Priority level | Optional. | + +### Client connections actively streaming + +| Client connections actively streaming | | +| ------------------------------------- | ------------------------------------------------------------ | +| Metric | sum by (instance) (typha_connections_streaming) | +| Example value | \{instance="10.0.1.20:9093"\} 10
\{instance="10.0.1.31:9093"\} 5 | +| Explanation | Current number of active connections that are "streaming" (have completed the handshake), to this Typha. After a connection has been Accepted (reported in the previous metric), there will be a handshake before the connection is deemed to be actively streaming. This indicates how many clients are connected to a Typha. The sum reflects per-cache metrics as well. | +| Threshold value recommendation | Compare the value for Total Connections Accepted and Client Connections Actively Streaming. The fluctuation of these values should be in-sync with each other if Accepted Connections are turning into Actively Streamed connections. If there is a discrepancy , you should investigate. Note: As always, it is recommended to baseline the relationship between these two metrics to have a sense of what is normal. It is also worth noting that in smaller clusters, it is normal for Typha to be unbalanced. Typha can handle hundreds of connections so it is of no concern if all nodes in a 10-node cluster (for example) connect to the same Typha. | +| Threshold breach symptoms | Felix is not getting updates from Typha. $[prodname] network policies are out-of-sync. | +| Threshold breach recommendations | Check Typha and Felix logs, and rollout restart Typha(s) if needed. | +| Priority level | Recommended. | + +### Rebalanced client connections + +| Rebalanced client connections | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | rate(typha_connections_dropped\{$_rate_interval\}) | +| Example value | \{endpoint="metrics-port", instance="10.0.1.20:9093", job="typha-metrics-svc", namespace="calico-system", pod="calico-typha-6c6cc9fcf7-csbdl", service="typha-metrics-svc"\} | +| Explanation | Number of client connections dropped to rebalance and share the load across different Typhas. | +| Threshold value recommendation | It is normal to see this value increasing sometimes. Investigate if connection dropped counters is rising constantly. If all Typhas are dropping connections because all Typhas believe they have too much load, this also warrants investigation. | +| Threshold breach symptoms | Dropping connections is rate limited so it should not affect the cluster as a whole. Typha clients, like Felix, will get dropped sometimes (but not constantly), and could result in periodic delays to policy updates. | +| Threshold breach recommendations | Ensure that the Kubernetes nodes have enough resources. | +| Priority level | Optional. | + +### 99 percentile client fall-behind + +| 99 percentile client fall-behind | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | max by (instance) (typha_client_latency_secs\{quantile='0.99'\}) | +| Example value | \{instance="10.0.1.20:9093"\} 0.1234
\{instance="10.0.1.31:9093"\} 0.1234 | +| Explanation | This metric measures how far behind Typha's client-handling threads are at reading updates.This metric will increase if:
a) The client (e.g Felix) is slow or overloaded and cannot keep up with what Typha is sending or
b) Typha is overloaded and it cannot keep up with writes to all its clients.

This metric is a good indication of your cluster, Felix, and Typha health. | +| Threshold value recommendation | It is normal for this to spike when new clients connect; they must download and process the snapshot, during which time they will fall slightly behind. Investigate of latency persists. | +| Threshold breach symptoms | Typha clients receiving updates from Typha will be behind in time. Potential symptoms could include $[prodname] network policies being out-of-sync. | +| Threshold breach recommendations | Check Typha and Felix logs and resource usage. It is recommended to focus on Felix logs and resource usage first, as there is generally more overhead with Felix and thus more of a chance of overload. Rollout restart Typha(s) and calico-node(s) if needed. | +| Priority level | Recommended. | + +### 99 percentile client write latency + +| 99 percentile client write latency | | +| ---------------------------------- | ------------------------------------------------------------ | +| Metric | max by (instance) (typha_client_write_latency_secs) | +| Example value | \{instance="10.0.1.20:9093"\} 0.007450815 | +| Explanation | Time for Typha to write to a client's socket (for example, Felix). | +| Threshold value recommendation | If the write latency is increasing, this indicates that a client (for example, Felix) is having an issue, or the network is having an issue. It is normal for intermittent spikes. Investigate any persistent latency. | +| Threshold breach symptoms | Typha clients will lag behind in receiving updates that Typha is sending. Potential symptoms include $[prodname] network policies being out-of-sync. | +| Threshold breach recommendations | Check Felix logs and resource usage. | +| Priority level | Recommended. | + +### 99 percentile client ping latency + +| 99 percentile client ping latency | | +| --------------------------------- | ------------------------------------------------------------ | +| Metric | max by (instance) (typha_ping_latency\{quantile="0.99"\}) | +| Example value | \{instance="10.0.1.20:9093"\} 0.034285331 | +| Explanation | This metric tracks the round-trip-time from Typha to a client. How long it takes for Typha's clients to respond to pings over the Typha protocol. | +| Threshold value recommendation | An increase in this metric above 1 second indicates that the clients, network or Typha are more heavily loaded. It is normal for intermittent spikes. Persistent latency above 1 second warrants investigation. | +| Threshold breach symptoms | Typha clients could be behind in time on updates Typha is sending. Potential symptoms include $[prodname] network policies being out-of-sync. | +| Threshold breach recommendations | Check Typha and Felix logs and resource usage. It is recommended to focus on Felix logs and resource usage first, as there is generally more overhead with Felix and thus more of a chance of overload. Check if the node is overloaded and review/increase calico-node/Typha CPU requests if needed. If needed, rollout restart Typha(s) and calico-node(s). | +| Priority level | Recommended. | + +## Typha cache internals + +### 99 percentile breadcrumb size + +| 99 percentile breadcrumb size | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | max by (instance) (typha_breadcrumb_size\{quantile="0.99"\}) | +| Explanation | Typha stores datastore changes as a series of blocks called breadcrumbs. Typha will store updates inside of these breadcrumbs (for example if a pod churned, this would be a single update). Typha can store multiple updates in a single breadcrumb with the default maximum size number being 100. | +| Threshold value recommendation | Typha generating blocks of size 100 during start up is normal. Investigate if Typha is consistently generating blocks of size 90+, which can indicate Typha is overloaded. | +| Threshold breach symptoms | Maintained block of sizes of 100 can indicate that Typha is falling behind on information and updates contained in the datastore. This will lead to Typha clients also falling behind (for example, $[prodname] network policy object may not be current). | +| Threshold breach recommendations | Check Typha logs and resource usage. Check if there is a lot of activity within the cluster that would cause Typha to send large breadcrumbs (for example, a huge amount of pod churn). If possible, reduce churn rate of resources on the cluster. | +| Priority level | Recommended. | + +### Non-blocking breadcrumbs fraction + +| Non-blocking breadcrumb fraction | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | (sum by (instance) (rate(typha_breadcrumb_non_block\{30s\})))/((sum by (instance) (rate(typha_breadcrumb_non_block\{30s\})))+(sum by (instance) (rate(typha_breadcrumb_block\{30s\})))) | +| Example value | \{instance="10.0.1.20:9093"\} NaN | +| Explanation | Typha stores datastore changes as a series of blocks called "breadcrumbs". Each client "follows the breadcrumbs" either by blocking and waiting, or skipping to the next one (non-blocking) if it is already available. Non-blocking breadcrumb actions indicates that Typha is constantly sending breadcrumbs to keep up with the datastore. Blocking breadcrumb actions indicate that Typha and the client have caught up, are up-to-date, and are waiting on the next breadcrumb. This metric will give a ratio between blocking and non-blocking actions that can indicate the health of Typha, its clients, and the cluster. | +| Threshold value recommendation | As the load on Typha increases, the ratio of skip-ahead, non-blocking reads, increases. If it approaches 100% then Typha may be overloaded (since clients only do non-blocking reads when they're behind). | +| Threshold breach symptoms | Consistent non-blocking breadcrumbs could indicate that Typha is falling behind on information and updates contained in the datastore. This will lead to Typha clients also being behind (for example, $[prodname] network policy object may not be current). | +| Threshold breach recommendations | Check Typha and Felix logs and resource usage. Check if there is a lot of activity within the cluster that would cause Typha to continuously send non-blocking breadcrumbs. | +| Priority level | Recommended. | + +### Datastore updates total + +| Datastore updates total | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | sum by (instance) (rate(typha_updates_total\{30s\})) | +| Example value | \{instance="10.0.1.20:9093"\} 0 | +| Explanation | The rate of updates from the datastore(s). For example, updates to Pods/Nodes/Policies/etc. | +| Threshold value recommendation | Intermittent spikes are expected. Constant updates indicates a very busy cluster (for example, lots of pod churn). | +| Threshold breach symptoms | Constant updates could lead to overloaded Typhas whereTyphas clients could fall behind. | +| Threshold breach recommendations | Ensure Typha has enough resources to handle a very dynamic cluster. | +| Priority level | Optional. | + +### Datastore update skipped (no-ops) + +| Datastore update skipped (no-ops) | | +| --------------------------------- | ------------------------------------------------------------ | +| Metric | sum by (instance) (rate(typha_updates_skipped\{30s\})) | +| Example value | \{instance="10.0.1.20:9093"\} 0 | +| Explanation | The number of updates from the datastore that Typha detected were no-ops. For example, an update to a Kubernetes node resource that did not touch any values that is of interest to $[prodname]. Such updates are not propagated to clients, which saves resources. | +| Threshold value recommendation | N/A | +| Threshold breach symptoms | N/A | +| Threshold breach recommendations | N/A | +| Priority level | Optional. | + +## Typha snapshot details + +### Snapshot send time + +| Median snapshot send time | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | max by (instance) (typha_client_snapshot_send_secs\{quantile="0.5"\}) | +| Example value | \{instance="10.0.1.20:9093"\} NaN | +| Explanation | The median time to stream the initial datastore snapshot to each client. It is useful to know the time it takes for a client to receive the data when it connects; it does not include time to process the data. | +| Threshold value recommendation | Investigate if this value is moving towards 10s of seconds. | +| Threshold breach symptoms | High values of this metric could indicate that newly-started clients are taking a long time to get the latest snapshot of the datastore, increasing the window of time where networking/policy updates are not being applied to the data plane during a restart/upgrade. Typha has a write timeout for writing the snapshot; if a client cannot receive the snapshot within that timeout, it is disconnected. Clients falling behind on information and updates contained in the datastore (for example, $[prodname] network policy object may not be current). | +| Threshold breach recommendations | Check Typha and calico-node logs and resource usage. Check for network congestion. Investigate why a particular calico-node is slow; it is likely on an overloaded node with insufficient CPU). | +| Priority level | Optional. | + +### Clients requiring grace period + +| Clients requiring grace period | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | sum by (instance) (typha_connections_grace_used) | +| Example value | \{instance="10.0.1.20:9093"\} 0 | +| Explanation | The number of Typhas with clients that required a grace period. After sending the snapshot to the client, Typha allows a grace period for the client to catch up to the most recent data. Typha sending the initial snapshot should take < 1 second, but the processing of the snapshot could take longer, so this grace period is there to allow the newly connected client to process the snapshot. | +| Threshold value recommendation | If this metric is constantly increasing, it can indicate potential performance issues with Typha and clients. It can indicate that performance is being impacted and may warrant investigation. | +| Threshold breach symptoms | High values of this metric could indicate clients falling behind on information and updates contained in the datastore (for example, $[prodname] network policy object may not be current). | +| Threshold breach recommendations | Check Typha and calico-node logs and resource usage. Check for network congestion, and determine the root cause. | +| Priority level | Optional. | + +### Max snapshot size (raw) + +| Max snapshot size (raw) | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | max(typha_snapshot_raw_bytes) | +| Example value | \{\} 557359 | +| Explanation | The raw size in bytes of snapshots sent from Typha to clients. | +| Threshold value recommendation | N/A | +| Threshold breach symptoms | N/A | +| Threshold breach recommendations | N/A | +| Priority Level | Optional. | + +### Max snapshot size (compressed) + +| Max snapshot size (compressed) | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | max(typha_snapshot_compressed_bytes) | +| Example value | \{\}134845 | +| Explanation | The compressed size in bytes of snapshots sent from Typha to clients. | +| Threshold value recommendation | This metric can be helpful for customers to estimate the bandwidth requirements for Felix to startup. For example, if the compressed snapshot size is 20MB in size on average, and 1000 Felix/calico-nodes start up, the bandwidth requirements could be estimated at 20GB between the pool of Typha and the set of Felixes across the network. | +| Threshold breach symptoms | N/A | +| Threshold breach recommendations | N/A | +| Priority Level | Optional. | + +## Policy metrics + +:::note +The following policy metrics are a separate endpoint exposed by Felix that are used in the web console. They require special Prometheus configuration to scrape the metrics. For details, see [Policy metrics](./policy-metrics). + +::: + +### Denied traffic + +| Denied traffic | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | calico_denied_packets
calico_denied_bytes | +| Example value | calico_denied_packets\{endpoint="calico-metrics-port", instance="ip-10-0-1-30.ca-central-1.compute.internal", job="calico-node-metrics", namespace="calico-system", pod="calico-node-6pcqm", policy="default | +| Explanation | Number of packets or bytes that have been dropped by explicit or implicit deny rules. Note that you'll get one instance of `calico_denied_packets/bytes` for each policy rule that is denying traffic. For example: calico_denied_packets\{policy="tier1\|fv/policy1\|0\|deny\|-1",scrIP="10.245.13.133"\} | +| Threshold value recommendation | The general rule of thumb is this metric should report zero at a stable state. Any deviation means that policy and traffic have diverged. Achieving a zero state depends on the stability and maturity of your cluster and policy. | +| Threshold breach symptoms | Either unexpected traffic is being denied because of an attack (one example), or expected traffic is being denied because of a misconfiguration in a policy. | +| Threshold breach recommendations | If this metric indicates that policy and traffic have diverged, the recommended steps are: Determine if an attack is causing the metric to spike, or if these flows should be allowed. If the flow should indeed be allowed, update the policy or a preceding policy to allow this traffic. | +| Priority level | Recommended. | + +### Traffic per rule + +| Traffic per rule | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | cnx_policy_rule_bytes
cnx_policy_rule_packets | +| Example value | cnx_policy_rule_bytes\{action="allow", endpoint="calico-metrics-port", instance="ip-10-0-1-20.ca-central-1.compute.internal", job="calico-node-metrics", namespace="calico-system", pod="calico-node-qzpkt", policy="es-kube-controller-access", rule_direction="egress", rule_index="1", service="calico-node-metrics", tier="calico-system", traffic_direction="inbound"\} | +| Explanation | Number of bytes or packets handled by $[prodname] network policy rules. | +| Threshold value recommendation | This metric should usually be non-zero (unless expected). A zero value indicates the rule is not matching any packets, and could be surplus to requirements. | +| Threshold breach symptoms | N/A | +| Threshold breach recommendations | If this metrics consistently reports a zero value over an acceptable period of time, you can consider removing the policy rule. | +| Priority Level | Optional. | + +### Connections per policy rule + +| Connections per policy rule | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | cnx_policy_rule_connections | +| Example value | cnx_policy_rule_connections\{endpoint="calico-metrics-port", instance="ip-10-0-1-20.ca-central-1.compute.internal", job="calico-node-metrics", namespace="calico-system", pod="calico-node-qzpkt", policy="es-kube-controller-access", rule_direction="egress", rule_index="0", service="calico-node-metrics", tier="calico-system", traffic_direction="outbound"\} | +| Explanation | Number connections handled by $[prodname] policy rules. | +| Threshold value recommendation | This metric is similar to *Traffic per Rule* but this deals more with flow monitoring. This metric should usually be non-zero. A zero value indicates that the rule is not matching any packets and could be surplus to requirements. | +| Threshold breach symptoms | N/A | +| Threshold breach recommendations | If this metrics consistently reports a zero value over an acceptable period of time, this policy rule can be considered for removal. | +| Priority Level | Optional. | +## Felix cluster-state metrics + +### CPU usage + +| CPU usage | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | rate(process_cpu_seconds_total\{30s\}) \* 100 | +| Example value | \{endpoint="metrics-port", instance="10.0.1.20:9091", job="felix-metrics-svc", namespace="calico-system", pod="calico-node-qzpkt", service="felix-metrics-svc"\}3.1197504199664072 | +| Explanation | CPU in use by calico-node represented as a percentage of a core. | +| Threshold value recommendation | A spike at startup is normal. It is recommended to first achieve a baseline and then monitor for any unexpected increases from this baseline. Investigate if maintained CPU usage goes above 90%. | +| Threshold breach symptoms | Unexpected maintained CPU usage could cause Felix to fall behind and could cause delays to policy updates. | +| Threshold breach recommendations | Check CPU usage on Kubernetes nodes. Increase resources if needed, rollout restart calico-node(s) if needed. | +| Priority level | Recommended. | + +### Memory usage + +| Memory usage | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | process_resident_memory_bytes | +| Example value | process_resident_memory_bytes\{endpoint="metrics-port", instance="10.0.1.20:9091", job="felix-metrics-svc", namespace="calico-system", pod="calico-node-qzpkt", service="felix-metrics-svc"\} 98996224 | +| Explanation | Amount of memory in use by calico-node. | +| Threshold value recommendation | Recommended to achieve a baseline first, then monitor for any unexpected increases from this baseline. Investigate if maintained CPU usage goes above 90% of what is available from the underlying node. | +| Threshold breach symptoms | Unexpected, maintained, memory usage could cause Felix to fall behind and could cause delays to policy updates. | +| Threshold breach recommendations | Check memory usage on Kubernetes nodes. Increase resources if needed, rollout restart typha(s) if needed. | +| Priority level | Recommended. | + +### Active hosts on each endpoint + +| Active hosts on each endpoint | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | felix_active_local_endpoints | +| Example value | felix_active_local_endpoints\{endpoint="metrics-port", instance="10.0.1.30:9091", job="felix-metrics-svc", namespace="calico-system", pod="calico-node-6pcqm", service="felix-metrics-svc"\} 36 | +| Explanation | Number of active pod-networked pods, and HEPs, on this node. | +| Threshold value recommendation | Threshold relates to resource limits on the node for example kubelet's max pods setting. | +| Threshold breach symptoms | Suggests Felix is getting out of sync. | +| Threshold breach recommendations | Rolling restart calico-node and report issue to support. | +| Priority level | Optional. | + +### Active calico nodes + +| Active calico nodes | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | max(felix_cluster_num_hosts) | +| Example value | \{\} 3 | +| Explanation | Total number of nodes in the cluster that have calico-node deployed and running. | +| Threshold value recommendation | This value should be equal to the number of nodes in the cluster. If there are discrepancies, then calico-nodes on some nodes are having issues. | +| Threshold breach symptoms | $[prodname] network policies on affected nodes could be out-of-sync. | +| Threshold breach recommendations | Check calico-node logs, rollout restart calico-node if needed. | +| Priority level | Recommended. | + +### Felix cluster policies + +| Felix cluster policies | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | felix_cluster_num_policies | +| Example value | felix_cluster_num_policies\{endpoint="metrics-port", instance="10.0.1.20:9091", job="felix-metrics-svc", namespace="calico-system", pod="calico-node-qzpkt", service="felix-metrics-svc"\} 58 | +| Explanation | Total number of $[prodname] network policies in the cluster. | +| Threshold value recommendation | Because $[prodname] is a distributed system, the number of policies should be generally consistent across all nodes. It is expected to have some skew between nodes for a short period of time while they sync, however they should never be out of sync for very long. | +| Threshold breach symptoms | If nodes are out of sync for long time, calico-nodes may be having issues or experiencing resource contention. Check the Errors Plot to see if there are any iptables errors reported. | +| Threshold breach recommendations | Redeploy calico-node if issues are seen, and increase resources if needed. | +| Priority level | Optional. | + +### Felix active local policies + +| Felix active local policies | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | felix_active_local_policies | +| Example value | felix_active_local_policies\{endpoint="metrics-port", instance="10.0.1.30:9091", job="felix-metrics-svc", namespace="calico-system", pod="calico-node-6pcqm", service="felix-metrics-svc"\} 44 | +| Explanation | Total number of network policies deployed on per node basis. | +| Threshold value recommendation | There is no hard limit to active policies. We can handle 1000+ active policies, but it impacts performance, especially if there's pod churn. The best solution is to optimize policies by combining multiple rules into one policy, and make sure that top-level policy selectors are being used. | +| Threshold breach symptoms | N/A | +| Threshold breach recommendations | Redeploy calico-node if issues are seen, and increase resources if needed. | +| Priority level | Recommended. | + +### Felix open FDS + +| Felix open FDS | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | sum by (pod) (process_open_fds\{pod=~"calico-node.*"\}) | +| Example value | \{pod="calico-node-6pcqm"\} 90 | +| Explanation | Number of opened file descriptors per calico-node pod. | +| Threshold value recommendation | Alert on this metric when it approaches the ulimit (as reported in `process_max_fds` value). You should not be anywhere near the maximum. | +| Threshold breach symptoms | Felix may become unstable/crash or fail to apply updates as it should. These failures and issues are logged. | +| Threshold breach recommendations | Check Felix logs, redeploy calico-node if you see log issues, and increase `max_fds value` if possible. | +| Priority Level | Optional. | + +### Felix max FDS + +| Felix max FDS | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | sum by (pod) (process_max_fds\{pod=~"calico-node.*"\}) | +| Example value | \{pod="calico-node-qzpkt"\} 1048576 | +| Explanation | Maximum number of opened file descriptors allowed per calico-node pod. | +| Threshold value recommendation | N/A | +| Threshold breach symptoms | N/A | +| Threshold breach recommendations | N/A | +| Priority level | Optional. | + +### Felix resync started + +| Felix resync started | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | sum(rate(felix_resyncs_started\{5m\})) | +| Explanation | This is the number of times that Typha has reported to Felix that it is re-connecting with the datastore. | +| Threshold value recommendation | Occasional resyncs are normal. Investigate resync counters that rapidly rise. | +| Threshold breach symptoms | Typha pods may be having issues or experiencing resource contention. Some calico-nodes that are paired with Typha pods experiencing issues will not be able to sync with the datastore. | +| Threshold breach recommendations | Investigate the root cause to avoid redeploying Typha (which can be very disruptive). Check resource contention and network connectivity from Typha to the datastore to see if Typha is working fine or if the API server is overloaded. | +| Priority level | Recommended. | + +### Felix dropped logs + +| Felix dropped logs | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | felix_logs_dropped | +| Example value | felix_logs_dropped\{endpoint="metrics-port", instance="10.0.1.20:9091", job="felix-metrics-svc", namespace="calico-system", pod="calico-node-qzpkt", service="felix-metrics-svc"\} 0 | +| Explanation | The number of logs Felix has dropped. Note that this metric does not count flow-logs; it counts logs to stdout. | +| Threshold value recommendation | Occasional drops are normal. Investigate if drop counters rapidly rise. | +| Threshold breach symptoms | Felix will drop logs if it cannot keep up with writing them out. These are ordinary code logs, not flow logs. Calico-node may be under resource constraints. | +| Threshold breach recommendations | Check CPU usage on calico-nodes and Kubernetes nodes. Increase resources if needed, and rollout restart calico-node(s) if needed. | +| Priority level | Optional. | + +## Felix error metrics + +### IPset errors + +| IPset errors | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | sum(rate(felix_ipset_errors\{5m\})) | +| Example value | \{\} 0 | +| Explanation | Number of ipset creation, modification, and deletion command failures. This metric reports how many times the ipset command has failed when Felix tried to run it. An error can occur when Felix sends bad ipset command data, or the kernel throws an error (potentially because it was too busy to handle this request at that time). | +| Threshold value recommendation | Occasional errors are normal. Investigate error counters that rapidly rise. | +| Threshold breach symptoms | $[prodname] network policies may not scope all endpoints in network policy rules. Cluster nodes may be under resource contention, which may result in other _error and _seconds metrics rising. Repeated errors could mean some persistent problem (for example, some other process has created an IP set with that name, which is incompatible). | +| Threshold breach recommendations | See the Errors Plot graph to determine if the scope is cluster-wide or node-local. Check calico-node logs. Check resource usage and contention on Kubernetes nodes and calico-nodes. Add nodes/resources if needed. If resource contention is not seen, restart calico-node(s) and monitor. Ensure that other process using IPtables are not blocking $[prodname] network policy management. | +| Priority level | Optional. | + +### iptables restore errors + +| iptables restore errors | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | sum(rate(felix_iptables_restore_errors\{5m\})) | +| Explanation | The number of iptables-restore errors over five minutes. The iptables-restore command is used when $[prodname] makes a change to iptables. For example, a new WEP or HEP is created, changes to a WEP or HEP or a change to a policy that affects a WEP or HEP. | +| Threshold value recommendation | Occasional errors are normal. Investigate error counters that rapidly rise. | +| Threshold breach symptoms | $[prodname] network policies are not up to date. Cluster nodes may be under resource contention, which may result in other _error and _seconds metrics rising. | +| Threshold breach recommendations | See the Errors Plot graph to determine if the scope is cluster-wide or node-local. Check calico-node logs. Check resource usage and contention on Kubernetes nodes and calico-nodes. Add nodes/resources if needed. If no resource contention is seen, restart calico-node and monitor. | +| Priority level | Optional. | + +### iptables save errors + +| iptables save errors | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | sum(rate(felix_iptables_save_errors\{5m\})) | +| Example value | \{\} 0 | +| Explanation | Number of iptables-save errors. The iptables-save command is run before every iptables-restore command so that $[prodname] has the current state of iptables. | +| Threshold value recommendation | Occasional errors are normal. Investigate error counters that rapidly rise. | +| Threshold breach symptoms | $[prodname] network policies are not up to date. Cluster nodes may be under resource contention, which may result in other _error and _seconds metrics rising. Repeated errors could mean some persistent problem (for example, some other process has creating iptables rules that $[prodname] cannot decode with the version of iptables-save in use). | +| Threshold breach recommendations | See the Errors Plot graph to determine if the scope is cluster-wide or node-local. Check calico-node logs. Check resource usage and contention on Kubernetes nodes and calico-nodes. Add nodes/resources if needed. If no resource contention is seen, restart calico-node and monitor. | +| Priority level | Optional. | + +### Felix log errors + +| Felix log errors | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | sum(rate(felix_log_errors\{5m\})) | +| Example value | \{\} 0 | +| Explanation | The number of times Felix fails to write out a log because the log buffer is full. | +| Threshold value recommendation | Occasional errors are normal. Investigate error counters that rapidly rise. | +| Threshold breach symptoms | Calico-node may be under resource contention, which may result in other _error and _seconds metrics rising. | +| Threshold breach recommendations | See the Errors Plot graph to determine if the scope is cluster-wide or node-local. Check resource usage and contention on Kubernetes nodes and calico-nodes. Add nodes/resources if needed. If no resource contention is seen, restart calico-node and monitor. | +| Priority level | Optional. | + +### Monitor Felix metrics using a graph + +| Errors plot graph | | +| -------------------------------- | ------------------------------------------------------------ | +| Metric | rate(felix_ipset_errors\{5m\}) \|\| rate(felix_iptables_restore_errors[5m]) \|\| rate(felix_iptables_save_errors[5m]) \|\| rate(felix_log_errors\{5m\}) | +| Example value | \{endpoint="metrics-port", instance="10.0.1.20:9091", job="felix-metrics-svc", namespace="calico-system", pod="calico-node-qzpkt", service="felix-metrics-svc"\} 0 | +| Explanation | Checks if there have been any iptables-save, iptables-restore, or ipset command errors in the past five minutes. Keeps track of what node is reporting which error. | +| Threshold value recommendation | Occasional errors are normal. Investigate error counters that rapidly rise. For this specific metric it is worth focusing on the metric that is spiking, and referencing that metric information. | +| Threshold breach symptoms | Dependent on the specific metric that is logging errors. | +| Threshold breach recommendations | If more than one metric is rising, check if all rising metrics are related to a specific calico-node. If this is the case, then the issue is local to that calico-node. Check calico-node logs. Check resource usage for the node and calico-node pod. If more than one metric is rising rapidly across all calico-nodes, then it is a cluster-wide issue and cluster health must be checked. Check cluster resource usage, cluster networking/infrastructure health, and restart calico-nodes and calico-typha pods. | +| Priority level | Recommended. | + +## Felix time-based metrics + +### Data plane apply time quantile 0.5/0.9/0.99 + +| Data plane apply time quantile 0.5/0.9/0.99 | | +| ------------------------------------------ | ------------------------------------------------------------ | +| Metric | felix_int_dataplane_apply_time_seconds\{quantile="0.5"\}
felix_int_dataplane_apply_time_seconds\{quantile="0.9"\}
felix_int_dataplane_apply_time_seconds\{quantile="0.99"\} | +| Example value | felix_int_dataplane_apply_time_seconds\{quantile="0.5"\}:felix_int_dataplane_apply_time_seconds\{endpoint="metrics-port", instance="10.0.1.30:9091", job="felix-metrics-svc", namespace="calico-system", pod="calico-node-6pcqm", quantile="0.5", service="felix-metrics-svc"\} 0.020859218 | +| Explanation | Time in seconds that it took to apply a data plane update ,viewed at the median, 90th percentile, and 99th percentile. | +| Threshold value recommendation | Thresholds will vary depending on cluster size and rate of churn. It is recommended that a baseline be set to determine a normal threshold value. In the field we have seen >10s in extremely high-scale clusters with 100k+ endpoints and lots of policy/Kubernetes services. | +| Threshold breach symptoms | Large time-to-apply values will cause a delay between $[prodname] network policy commits and enforcement in the data plane. This is dependent on how $[prodname] waiting for kube-proxy to release the iptables lock, which is influenced by the number of services in use. | +| Threshold breach recommendations | Increase cluster resources, and reduce the number of Kubernetes services if possible. | +| Priority level | Recommended. | + +### Felix route table list seconds quantile 0.5/0.9/0.99 + +| Felix route table list seconds quantile 0.5/0.9/0.99 | | +| ---------------------------------------------------- | ------------------------------------------------------------ | +| Metric | felix_route_table_list_seconds\{quantile="0.5"\}
felix_route_table_list_seconds\{quantile="0.9"\}
felix_route_table_list_seconds\{quantile="0.99"\} | +| Example value | felix_route_table_list_seconds\{quantile="0.5"\}:felix_route_table_list_seconds\{endpoint="metrics-port",instance="10.0.1.30:9091",job="felix-metrics-svc",namespace="calico-system", pod="calico-node-6pcqm",quantile="0.5", service="felix-metrics-svc"\} 0.000860426 | +| Explanation | Time to list all the interfaces during a resync, viewed at the median, 90th percentile and 99th percentile. | +| Threshold value recommendation | Thresholds will vary depending on the number of cali interfaces per node. It is recommended that a baseline be set to determine a normal threshold value. | +| Threshold breach symptoms | High values indicate high CPU usage in felix and slow data plane updates. | +| Threshold breach recommendations | Increase cluster resources. Reduce the number of cali interfaces per node where possible. | +| Priority level | Optional. | + +### Felix graph update time quantile 0.5/0.9/0/99 + +| Felix graph update time seconds quantile 0.5/0.9/0.99 | | +| ----------------------------------------------------- | ------------------------------------------------------------ | +| Metric | felix_calc_graph_update_time_seconds\{quantile="0.5"\}
felix_calc_graph_update_time_seconds\{quantile="0.9"\}
felix_calc_graph_update_time_seconds\{quantile="0.99"\} | +| Example value | felix_calc_graph_update_time_seconds\{quantile="0.5"\}:felix_calc_graph_update_time_seconds\{endpoint="metrics-port",instance="10.0.1.30:9091", job="felix-metrics-svc",namespace="calico-system", pod="calico-node-6pcqm",quantile="0.5", service="felix-metrics-svc"\} 0.00007129 | +| Explanation | This metric reports the time taken to update the calculation graph for each datastore on an update call, viewed at the median, 90th percentile and 99th percentile. The calculation graph is the Felix component that takes all the policies/workload endpoints/host endpoints information that it has received from Typha, and distills it down to data plane updates that are relevant for this node. | +| Threshold value recommendation | After *start of day* (where we will typically get a large update), then values should be sub 1 second (with occasional blips to 1+ seconds). Should be measured in milliseconds with the occasional blip to a second or two. Investigate if the result is constantly in values of seconds. | +| Threshold breach symptoms | High values indicate high CPU usage in felix and slow data plane updates. | +| Threshold breach recommendations | Increase cluster resources. Check calico-node logs. Rollout restart calico-node(s) if needed. | +| Priority level | Recommended. | diff --git a/calico-cloud_versioned_docs/version-24-1/operations/monitor/prometheus/alertmanager.mdx b/calico-cloud_versioned_docs/version-24-1/operations/monitor/prometheus/alertmanager.mdx new file mode 100644 index 0000000000..49b5fb329c --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/monitor/prometheus/alertmanager.mdx @@ -0,0 +1,103 @@ +--- +description: Configure Alertmanager in a Calico Cloud connected cluster to route Prometheus alerts to operators with deduplication, grouping, silencing, and inhibition rules. +--- + +# Configure Alertmanager + +Alertmanager is used by $[prodname] to route alerts from Prometheus to the administrators. +It handles routing, deduplicating, grouping, silencing and inhibition of alerts. + +More detailed information about Alertmanager is available in the [upstream documentation](https://prometheus.io/docs/alerting/latest/configuration). + +### Updating the Alertmanager config + +- Save the current alertmanager secret, usually named `alertmanager-`. + Our manifests will end up creating a secret called: `alertmanager-calico-node-alertmanager`. + + ```bash + kubectl -n tigera-operator get secrets alertmanager-calico-node-alertmanager -o yaml > alertmanager-secret.yaml + ``` + +- The current alertmanager.yaml file is encoded and stored inside the + `alertmanager.yaml` key under the `data` field. You can decode it by + copying the value of `alertmanager.yaml` and using the `base64` command. + + ```bash + echo "" | base64 --decode > alertmanager-config.yaml + ``` + +- Make necessary changes to `alertmanager-config.yaml`. Once this is done, + you have to re-encode and save it to `alertmanager-secret.yaml`. You can do + this by (in Linux): + + ```bash + cat alertmanager-config.yaml | base64 -w 0 + ``` + +- Paste the output of the running the command above back in `alertmanager-secret.yaml` + replacing the value present in `alertmanager.yaml` field. Then apply this + updated manifest. + + ```bash + kubectl -n tigera-operator apply -f alertmanager-secret.yaml + ``` + +Your changes should be applied in a few seconds by the config-reloader +container inside the alertmanager pod launched by the prometheus-operator +(usually named `alertmanager-`). + +For more advice on writing alertmanager configuration files, see the +[alertmanager configuration](https://prometheus.io/docs/alerting/latest/configuration/) documentation. + +### Configure Inhibition Rules + +Alertmanager has a feature to suppress certain notifications according to +defined rules. A typical use case for defining `inhibit` rules is to suppress +notifications from a lower priority alert when one with a higher priority is +firing. These inhibition rules are defined in the alertmanager configuration +file. You can define one by adding this configuration snippet to your +`alertmanager.yaml`. + +```yaml noValidation +[...] +inhibit_rules: +- source_match: + severity: 'critical' + target_match: + severity: 'info' + # Apply inhibition for alerts generated by the same alerting rule + # and on the same node. + equal: ['alertname', 'instance'] +[...] +``` + +### Configure Grouping of Alerts + +Alertmanager also has a feature to group alerts based on labels and fine tune +how often to resend an alert and so on. In the case of Denied Packet metrics, +simply defining a Prometheus alerting rule would mean that you will get an +page (if so defined in your alertmanager configuration) for every policy on +every node for every Source IP. All these alerts can be combined into a single +alert by configuring grouping. The Alertmanager configuration file that is +provided with $[prodname] by default, groups alerts on a +per-node basis. Instead, if the goal is to group all alerts with the same +name, edit (and apply) the alertmanager configuration file like so: + +```yaml +global: + resolve_timeout: 5m +route: + group_by: ['alertname'] + group_wait: 30s + group_interval: 1m + repeat_interval: 5m + receiver: 'webhook' +receivers: + - name: 'webhook' + webhook_configs: + - url: 'http://calico-alertmanager-webhook:30501/' +``` + +More information, including descriptions of the various options can be found under the +[route section](https://prometheus.io/docs/alerting/latest/configuration/#route) +of the Alertmanager Configuration guide. diff --git a/calico-cloud_versioned_docs/version-24-1/operations/monitor/prometheus/byo-prometheus.mdx b/calico-cloud_versioned_docs/version-24-1/operations/monitor/prometheus/byo-prometheus.mdx new file mode 100644 index 0000000000..1cb58e9e9b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/monitor/prometheus/byo-prometheus.mdx @@ -0,0 +1,435 @@ +--- +description: Scrape Calico Cloud component metrics from an existing bring-your-own Prometheus deployment instead of the bundled operator-managed Prometheus on connected clusters. +--- + +# Bring your own Prometheus + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +## Big picture + +Scrape $[prodname] metrics for Bring Your Own (BYO) Prometheus. + +## Value + +$[prodname] uses the Prometheus monitoring tool to scrape metrics from instrumented jobs, and displays time-series data in a visualizer such as Grafana. You can scrape the following time-series metrics for $[prodname] components to your own Prometheus: + +- elasticsearch +- fluentd +- calico-node +- kube-controllers +- felix +- typha (not enabled by default) +- operator + +## Before you begin + +**Supported** + +For the supported version of Prometheus in this release, see the [Release Notes](../../../release-notes/index.mdx) (`coreos-prometheus`). + +## How to + +- [Scrape all enabled metrics](#scrape-all-enabled-metrics) +- [Scrape metrics from specific components directly](#scrape-metrics-from-specific-components-directly) +- [Verify BYO Prometheus](#verify-byo-prometheus) +- [Create policy to secure traffic between pods](#create-policy-to-secure-traffic-between-pods) +- [Troubleshooting](#troubleshooting) + +### Scrape all enabled metrics + +In this section we create a service monitor that scrapes all enabled metrics. To enable metrics that +are not enabled by default, please consult the [next section](#scrape-metrics-from-specific-components-directly). + +The following example shows a Prometheus server installed in namespace "external-prometheus" with a `serviceMonitorSelector` that selects all service monitors with the label `k8s-app=tigera-external-prometheus`. + +1. Save the following configuration in a file called `monitor.yaml`. + + ```yaml + apiVersion: operator.tigera.io/v1 + kind: Monitor + metadata: + name: tigera-secure + spec: + externalPrometheus: + namespace: external-prometheus + serviceMonitor: + labels: + k8s-app: tigera-external-prometheus + ``` + For a list of all configuration options, see the [Tigera Operator API reference](../../../reference/installation/api.mdx). + +2. Apply the manifest to your cluster. + + ```bash + kubectl apply -f monitor.yaml + ``` + +3. Verify that the new configuration has been added to your cluster + ```bash + export NS=external-prometheus + kubectl get servicemonitor -n $NS tigera-external-prometheus + kubectl get serviceaccount -n $NS tigera-external-prometheus + kubectl get secret -n $NS tigera-external-prometheus + kubectl get clusterrole tigera-external-prometheus + kubectl get clusterrolebinding tigera-external-prometheus + ``` + That's it. You should be seeing the new metrics show up in your Prometheus instance within a minute. For more information on verifying metrics, see the section, [Verify BYO Prometheus](#verify-byo-prometheus). + +### Scrape metrics from specific components directly + +We recommend the previous section for scraping all enabled metrics. Read on if you wish to scrape metrics from specific +components directly using mTLS, or if you wish to enable metrics that are disabled by default. + + + + +**Configure TLS certificates** + +1. Copy the required secret and configmap to your namespace. +2. Save the manifest of the required TLS secret and CA configmap. + + ```bash + kubectl get secret calico-node-prometheus-client-tls -n tigera-prometheus -o yaml > calico-node-prometheus-client-tls.yaml + ``` + + ```bash + kubectl get configmap -n tigera-prometheus tigera-ca-bundle -o yaml > tigera-ca-bundle.yaml + ``` + +3. Edit `calico-node-prometheus-client-tls.yaml` and `tigera-ca-bundle.yaml` by changing the namespace to the namespace where your prometheus instance is running. +4. Apply the manifests to your cluster. + + ```bash + kubectl apply -f calico-node-prometheus-client-tls.yaml + ``` + + ```bash + kubectl apply -f tigera-ca-bundle.yaml + ``` + +**Create the service monitor** + +Apply the ServiceMonitor to the namespace where Prometheus is running. + +```bash +export NAMESPACE= +``` + +```bash +kubectl apply -f $[filesUrl_CE]/manifests/prometheus/elasticsearch-metrics-service-monitor.yaml -n $NAMESPACE +``` + +The .yamls have no namespace defined so when you apply `kubectl`, it is applied in the $NAMESPACE. + + + + +**Configure TLS certificates** + +1. Copy the required secret and configmap to your namespace. +2. Save the manifest of the required TLS secret and CA configmap. + + ```bash + kubectl get secret calico-node-prometheus-client-tls -n tigera-prometheus -o yaml > calico-node-prometheus-client-tls.yaml + ``` + + ```bash + kubectl get configmap -n tigera-prometheus tigera-ca-bundle -o yaml > tigera-ca-bundle.yaml + ``` + +3. Edit `calico-node-prometheus-client-tls.yaml` and `tigera-ca-bundle.yaml` and change the namespace to the namespace where your prometheus instance is running. +4. Apply the manifests to your cluster. + + ```bash + kubectl apply -f calico-node-prometheus-client-tls.yaml + ``` + + ```bash + kubectl apply -f tigera-ca-bundle.yaml + ``` + +**Create the service monitor** + +Apply the ServiceMonitor to the namespace where Prometheus is running. + +```bash +export NAMESPACE= +``` + +```bash +kubectl apply -f $[filesUrl_CE]/manifests/prometheus/fluentd-metrics-service-monitor.yaml -n $NAMESPACE +``` + +The .yamls have no namespace defined so when you apply `kubectl`, it is applied in the $NAMESPACE. + + + + +**Configure TLS certificates** + +1. Copy the required secret and configmap to your namespace. +2. Save the manifest of the required TLS secret and CA configmap. + + ```bash + kubectl get secret calico-node-prometheus-client-tls -n tigera-prometheus -o yaml > calico-node-prometheus-client-tls.yaml + ``` + + ```bash + kubectl get configmap -n tigera-prometheus tigera-ca-bundle -o yaml > tigera-ca-bundle.yaml + ``` + +3. Edit `calico-node-prometheus-client-tls.yaml` and `tigera-ca-bundle.yaml` by changing the namespace to the namespace where your prometheus instance is running. +4. Apply the manifests to your cluster. + + ```bash + kubectl apply -f calico-node-prometheus-client-tls.yaml + ``` + + ```bash + kubectl apply -f tigera-ca-bundle.yaml + ``` + +**Create the service monitor** + +Apply the ServiceMonitor to the namespace where Prometheus is running. + +```bash +export NAMESPACE= +``` + +```bash +kubectl apply -f $[filesUrl_CE]/manifests/prometheus/calico-node-monitor-service-monitor.yaml -n $NAMESPACE +``` + +The .yamls have no namespace defined so when you apply `kubectl`, it is applied in $NAMESPACE. + + + + +**Configure TLS certificates** + +1. Copy the required secret and configmap to your namespace. +2. Save the manifest of the required TLS secret and CA configmap. + + ```bash + kubectl get secret calico-node-prometheus-client-tls -n tigera-prometheus -o yaml > calico-node-prometheus-client-tls.yaml + ``` + + ```bash + kubectl get configmap -n tigera-prometheus tigera-ca-bundle -o yaml > tigera-ca-bundle.yaml + ``` + +3. Edit `calico-node-prometheus-client-tls.yaml` and `tigera-ca-bundle.yaml` by changing the namespace to the namespace where your prometheus instance is running. +4. Apply the manifests to your cluster. + + ```bash + kubectl apply -f calico-node-prometheus-client-tls.yaml + ``` + + ```bash + kubectl apply -f tigera-ca-bundle.yaml + ``` + +**Create the service monitor** + +Apply the ServiceMonitor to the namespace where Prometheus is running. + +```bash +export NAMESPACE= +``` + +```bash +kubectl apply -f $[filesUrl_CE]/manifests/prometheus/kube-controller-metrics-service-monitor.yaml -n $NAMESPACE +``` + +The .yamls have no namespace defined so when you apply `kubectl`, it is applied in the $NAMESPACE. + + + + +**Enable metrics** + +Felix metrics are not enabled by default. + +By default, Felix uses **port 9091 TCP** to publish metrics. + +Use the following command to enable Felix metrics. + +```bash +kubectl patch felixconfiguration default --type merge --patch '{"spec":{"prometheusMetricsEnabled": true}}' +``` + +You should see a result similar to: + +``` +felixconfiguration.projectcalico.org/default patched +``` + +For all Felix configuration values, see [Felix configuration](../../../reference/component-resources/node/felix/configuration.mdx). + +For all Prometheus Felix configuration values, see [Felix Prometheus](../../../reference/component-resources/node/felix/prometheus.mdx). + +**For Windows nodes, create a service to expose Felix metrics** + +If you're running $[prodnameWindows], you must create a service to expose Felix metrics for Windows nodes: + +```bash +kubectl apply -f - < +``` + +```bash +kubectl apply -f $[filesUrl_CE]/manifests/prometheus/felix-metrics-service-monitor.yaml -n $NAMESPACE +``` + +The .yamls have no namespace defined so when you apply `kubectl`, it is applied in the $NAMESPACE. + + + + +**Enable metrics** + +Typha metrics are not enabled by default. + +By default, Typha uses **port 9091** TCP to publish metrics. However, if $[prodname] is installed using the Amazon yaml file, this port will be 9093 because it is set manually using the **TYPHA_PROMETHEUSMETRICSPORT** environment variable. + +Use the following command to enable Typha metrics. + +```bash +kubectl patch installation default --type=merge -p '{"spec": {"typhaMetricsPort":9093}}' +``` + +You should see a result similar to: + +```bash +installation.operator.tigera.io/default patched +``` + +**Create the service monitor** + +Apply the ServiceMonitor to the namespace where Prometheus is running. + +```bash +export NAMESPACE= +``` + +```bash +kubectl apply -f $[filesUrl_CE]/manifests/prometheus/typha-metrics-service-monitor.yaml -n $NAMESPACE +``` + +The .yamls have no namespace defined so when you apply `kubectl`, it is applied in the $NAMESPACE. + + + + +**Configure TLS certificates** + +1. Copy the required secret and configmap to your namespace. +2. Save the manifest of the required TLS secret and CA configmap. + + ```bash + kubectl get secret calico-node-prometheus-client-tls -n tigera-prometheus -o yaml > calico-node-prometheus-client-tls.yaml + ``` + + ```bash + kubectl get configmap -n tigera-prometheus tigera-ca-bundle -o yaml > tigera-ca-bundle.yaml + ``` + +3. Edit `calico-node-prometheus-client-tls.yaml` and `tigera-ca-bundle.yaml` by changing the namespace to the namespace where your prometheus instance is running. +4. Apply the manifests to your cluster. + + ```bash + kubectl apply -f calico-node-prometheus-client-tls.yaml + ``` + + ```bash + kubectl apply -f tigera-ca-bundle.yaml + ``` + +**Create the service monitor** + +Apply the ServiceMonitor to the namespace where Prometheus is running. + +```bash +export NAMESPACE= +``` + +```bash +kubectl apply -f $[filesUrl_CE]/manifests/prometheus/operator-metrics-service-monitor.yaml -n $NAMESPACE +``` + +The .yamls have no namespace defined so when you apply `kubectl`, it is applied in the $NAMESPACE. + + + + +### Verify BYO Prometheus + +1. Access the Prometheus dashboard using the port-forwarding feature. + + ```bash + kubectl port-forward pod/byo-prometheus-pod 9090:9090 -n $NAMESPACE + ``` + +1. Browse to the Prometheus dashboard: http://localhost:9090. + +1. In the Expression text box, enter your metric name and click the **Execute** button. + + The Console table is populated with all of your nodes with the number of endpoints. + +### Troubleshooting + +This section is applicable only if you experience issues with mTLS after following the [Scrape metrics from specific components directly](#scrape-metrics-from-specific-components-directly) +section. + +1. Extract the TLS credentials and CA bundle from the cluster. + + ```bash + kubectl get secret -n tigera-prometheus calico-node-prometheus-client-tls -o jsonpath='{.data.tls\.key}' | base64 -d > key.pem + kubectl get secret -n tigera-prometheus calico-node-prometheus-client-tls -o jsonpath='{.data.tls\.crt}' | base64 -d > cert.pem + kubectl get cm -n tigera-prometheus tigera-ca-bundle -o jsonpath='{.data.tigera-ca-bundle\.crt}' > bundle.pem + ``` + +1. Port-forward the Prometheus pods and run this command with the forwarded port. + + ```bash + curl --cacert bundle.pem --key key.pem --cert cert.pem https://localhost:8080/metrics + ``` + +You should be able to see the metrics. + +### Create policy to secure traffic between pods + +To support zero trust, we recommend that you create $[prodname] network policy to allow the traffic between BYO Prometheus pods, and the respective metrics pods. For samples of ingress and egress policies, see [Get started with Calico network policy](../../../network-policy/beginners/calico-network-policy.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/operations/monitor/prometheus/configure-prometheus.mdx b/calico-cloud_versioned_docs/version-24-1/operations/monitor/prometheus/configure-prometheus.mdx new file mode 100644 index 0000000000..b9b3579431 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/monitor/prometheus/configure-prometheus.mdx @@ -0,0 +1,342 @@ +--- +description: Configure Calico Cloud Prometheus rules for denied-packet alerts and persistent storage on connected clusters by editing PrometheusRule and StorageClass resources. +--- + +# Configure Prometheus + +## Updating denied packets rules + +This is an example of how to modify the sample rule created by the sample manifest. +The process of updating rules is the same as for user created rules (documented below). + +- Save the current alert rule: + + ```bash + kubectl -n tigera-prometheus get prometheusrule -o yaml > calico-prometheus-alert-rule-dp.yaml + ``` + +- Make necessary edits to the alerting rules then apply the updated manifest. + + ```bash + kubectl apply -f calico-prometheus-alert-rule-dp.yaml + ``` + +Your changes should be applied in a few seconds by the prometheus-config-reloader +container inside the prometheus pod launched by the prometheus-operator +(usually named `prometheus-`). + +As an example, the range query in this Manifest is 10 seconds. + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: calico + namespace: tigera-prometheus + labels: + role: tigera-prometheus-rules + prometheus: calico-node-prometheus +spec: + groups: + - name: calico.rules + rules: + - alert: DeniedPacketsRate + expr: rate(calico_denied_packets[10s]) > 50 + labels: + severity: critical + annotations: + summary: 'Instance {{$labels.instance}} - Large rate of packets denied' + description: '{{$labels.instance}} with calico-node pod {{$labels.pod}} has been denying packets at a fast rate {{$labels.sourceIp}} by policy {{$labels.policy}}.' +``` + +To update this alerting rule, to say, execute the query with a range of +20 seconds modify the manifest to this: + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: calico + namespace: tigera-prometheus + labels: + role: tigera-prometheus-rules + prometheus: calico-node-prometheus +spec: + groups: + - name: calico.rules + rules: + - alert: DeniedPacketsRate + expr: rate(calico_denied_packets[20s]) > 50 + labels: + severity: critical + annotations: + summary: 'Instance {{$labels.instance}} - Large rate of packets denied' + description: '{{$labels.instance}} with calico-node pod {{$labels.pod}} has been denying packets at a fast rate {{$labels.sourceIp}} by policy {{$labels.policy}}.' +``` + +## Creating a new alerting rule + +Creating a new alerting rule is straightforward once you figure out what you +want your rule to look for. Check [alerting rules](https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) +and [Queries](https://prometheus.io/docs/querying/examples/) for more +information. + +### New alerting rule for monitoring Calico node + +To add the new alerting rule to our Prometheus instance, define a PrometheusRule manifest +in the `tigera-prometheus` namespace with the labels +`role: tigera-prometheus-rules` and `prometheus: calico-node-prometheus`. The +labels should match the labels defined by the `ruleSelector` field of the +Prometheus manifest. + +As an example, to fire a alert when a $[noderunning] instance has been down for +more than 5 minutes, save the following to a file, say `calico-node-down-alert.yaml`. + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: calico-prometheus-calico-node-down + namespace: tigera-prometheus + labels: + role: tigera-prometheus-rules + prometheus: calico-node-prometheus +spec: + groups: + - name: calico.rules + rules: + - alert: CalicoNodeInstanceDown + expr: up == 0 + for: 5m + labels: + severity: warning + annotations: + summary: 'Instance {{$labels.instance}} Pod: {{$labels.pod}} is down' + description: '{{$labels.instance}} of job {{$labels.job}} has been down for more than 5 minutes' +``` + +Then create/apply this manifest in kubernetes. + +```bash +kubectl apply -f calico-node-down-alert.yaml +``` + +Your changes should be applied in a few seconds by the prometheus-config-reloader +container inside the prometheus pod launched by the prometheus-operator +(usually named `prometheus-`). + +### New alerting rule for monitoring BGP peers + +Let’s look at an example of a new alerting rule to our Prometheus instance with respect to monitoring BGP +peering health. Define a PrometheusRule manifest in the tigera-prometheus namespace with the labels +`role: tigera-prometheus-rules` and `prometheus: calico-node-prometheus`. The labels should match the labels +defined by the `ruleSelector` field of the Prometheus manifest. + +As an example, to fire an alert when the number of peering connections with a status other than “Established” +is increasing at a non-zero rate in the cluster (over the last 5 minutes), save the following to a file, say +`tigera-peer-status-not-established.yaml`. + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + labels: + prometheus: calico-node-prometheus + role: tigera-prometheus-rules + name: tigera-prometheus-peer-status-not-established + namespace: tigera-prometheus +spec: + groups: + - name: calico.rules + rules: + - alert: CalicoNodePeerStatusNotEstablished + annotations: + description: '{{$labels.instance}} has at least one peer connection that is + no longer up.' + summary: Instance {{$labels.instance}} has peer connection that is no longer + up + expr: rate(bgp_peers{status!~"Established"}[5m]) > 0 + labels: + severity: critical +``` + +Then create/apply this manifest in kubernetes. + +```bash +kubectl apply -f tigera-peer-status-not-established.yaml +``` + +Your changes should be applied in a few seconds by the prometheus-config-reloader +container inside the prometheus pod launched by the prometheus-operator +(usually named `prometheus-`). + +## Additional Alerting Rules + +The Alerting Rules installed by the $[prodname] install manifest is a simple +one that fires an alert when the rate of denied packets denied by a policy on +a node from a particular Source IP exceeds a certain packets per second +threshold. The Prometheus query used for this (ignoring the threshold value 20) is: + +``` +rate(calico_denied_packets[10s]) +``` + +and this query will return results something along the lines of: + +``` +{endpoint="calico-metrics-port",instance="10.240.0.81:9081",job="calico-node-metrics",namespace="kube-system",pod="calico-node-hn0kl",policy="profile/k8s_ns.test/0/deny",service="calico-node-metrics",srcIP="192.168.167.129"} 0.6 +{endpoint="calico-metrics-port",instance="10.240.0.84:9081",job="calico-node-metrics",namespace="kube-system",pod="calico-node-97m3g",policy="profile/k8s_ns.test/0/deny",service="calico-node-metrics",srcIP="192.168.167.175"} 0.2 +{endpoint="calico-metrics-port",instance="10.240.0.84:9081",job="calico-node-metrics",namespace="kube-system",pod="calico-node-97m3g",policy="profile/k8s_ns.test/0/deny",service="calico-node-metrics",srcIP="192.168.252.157"} 0.4 +{endpoint="calico-metrics-port",instance="10.240.0.81:9081",job="calico-node-metrics",namespace="kube-system",pod="calico-node-hn0kl",policy="profile/k8s_ns.test/0/deny",service="calico-node-metrics",srcIP="192.168.167.175"} 1 +{endpoint="calico-metrics-port",instance="10.240.0.84:9081",job="calico-node-metrics",namespace="kube-system",pod="calico-node-97m3g",policy="profile/k8s_ns.test/0/deny",service="calico-node-metrics",srcIP="192.168.167.129"} 0.4 +{endpoint="calico-metrics-port",instance="10.240.0.81:9081",job="calico-node-metrics",namespace="kube-system",pod="calico-node-hn0kl",policy="profile/k8s_ns.test/0/deny",service="calico-node-metrics",srcIP="192.168.167.159"} 0.4 +{endpoint="calico-metrics-port",instance="10.240.0.81:9081",job="calico-node-metrics",namespace="kube-system",pod="calico-node-hn0kl",policy="profile/k8s_ns.test/0/deny",service="calico-node-metrics",srcIP="192.168.252.175"} 0.4 +{endpoint="calico-metrics-port",instance="10.240.0.84:9081",job="calico-node-metrics",namespace="kube-system",pod="calico-node-97m3g",policy="profile/k8s_ns.test/0/deny",service="calico-node-metrics",srcIP="192.168.252.175"} 0.6 +{endpoint="calico-metrics-port",instance="10.240.0.81:9081",job="calico-node-metrics",namespace="kube-system",pod="calico-node-hn0kl",policy="profile/k8s_ns.test/0/deny",service="calico-node-metrics",srcIP="192.168.252.157"} 0.6 +{endpoint="calico-metrics-port",instance="10.240.0.84:9081",job="calico-node-metrics",namespace="kube-system",pod="calico-node-97m3g",policy="profile/k8s_ns.test/0/deny",service="calico-node-metrics",srcIP="192.168.167.159"} 0.6 +``` + +We can modify this query to find out all packets dropped by different policies +on every node. + +``` +(sum by (instance,policy) (rate(calico_denied_packets[10s]))) +``` + +This query will aggregate the results from all different Source IPs, and +preserve the `policy` and `instance` labels. Note that the `instance` label +represents the calico node's IP Address and `PrometheusReporterPort`. This +query will return results like so: + +``` +{instance="10.240.0.84:9081",policy="profile/k8s_ns.test/0/deny"} 2 +{instance="10.240.0.81:9081",policy="profile/k8s_ns.test/0/deny"} 2.8 +``` + +To include the pod name in these results, add the label `pod` to the labels +listed in the `by` expression like so: + +``` +(sum by (instance,pod,policy) (rate(calico_denied_packets[10s]))) +``` + +which will return the following results: + +``` +{instance="10.240.0.84:9081",pod="calico-node-97m3g",policy="profile/k8s_ns.test/0/deny"} 2 +{instance="10.240.0.81:9081",pod="calico-node-hn0kl",policy="profile/k8s_ns.test/0/deny"} 2.8 +``` + +An interesting use case is when a rogue Pod is using tools such as nmap to +scan a subnet for open ports. To do this, we have to execute a query that will +aggregate across all policies on all instances while preserving the source IP +address. This can be done using this query: + +``` +(sum by (srcIP) (rate(calico_denied_packets[10s]))) +``` + +which will return results, different source IP address: + +``` +{srcIP="192.168.167.159"} 1.0000000000000002 +{srcIP="192.168.167.129"} 1.2000000000000002 +{srcIP="192.168.252.175"} 1.4000000000000001 +{srcIP="192.168.167.175"} 0.4 +{srcIP="192.168.252.157"} 1.0000000000000002 +``` + +To use these queries as Alerting Rules, follow the instructions defined in the +[Creating a new Alerting Rule](#creating-a-new-alerting-rule) section and create +a ConfigMap with the appropriate query. + +## Updating the scrape interval + +You may wish to modify the scrape interval (time between Prometheus polling each node for new denied packet information). +Increasing the interval reduces load on Prometheus and the amount of storage required, but decreases the detail of the collected metrics. + +The scrape interval of endpoints ($[noderunning] in our case) is defined as part of +the ServiceMonitor manifest. To change the interval: + +- Save the current ServiceMonitor manifest: + + ```bash + kubectl -n tigera-prometheus get servicemonitor calico-node-monitor -o yaml > calico-node-monitor.yaml + ``` + +- Update the `interval` field under `endpoints` to desired settings and + apply the updated manifest. + + ```bash + kubectl apply -f calico-node-monitor.yaml + ``` + +Your changes should be applied in a few seconds by the prometheus-config-reloader +container inside the prometheus pod launched by the prometheus-operator +(usually named `prometheus-`). + +As an example on what to update, the interval in this ServiceMonitor manifest +is 5 seconds (`5s`). + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: calico-node-monitor + namespace: tigera-prometheus + labels: + team: network-operators +spec: + selector: + matchLabels: + k8s-app: calico-node + namespaceSelector: + matchNames: + - kube-system + endpoints: + - port: calico-metrics-port + interval: 5s +``` + +To update $[prodname] Prometheus' scrape interval to 10 seconds modify the manifest +to this: + +```yaml +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: calico-node-monitor + namespace: tigera-prometheus + labels: + team: network-operators +spec: + selector: + matchLabels: + k8s-app: calico-node + namespaceSelector: + matchNames: + - kube-system + endpoints: + - port: calico-metrics-port + interval: 10s +``` + +## Troubleshooting Config Updates + +Check config reloader logs to see if they detected any recent activity. + +- For prometheus run: + + ```bash + kubectl -n tigera-prometheus logs prometheus- prometheus-config-reloader + ``` + +- For alertmanager run: + + ```bash + kubectl -n tigera-prometheus logs alertmanager- config-reloader + ``` + +The config-reloaders watch each pods file-system for updated config from +ConfigMap's or Secret's and will perform steps necessary for reloading +the configuration. diff --git a/calico-cloud_versioned_docs/version-24-1/operations/monitor/prometheus/index.mdx b/calico-cloud_versioned_docs/version-24-1/operations/monitor/prometheus/index.mdx new file mode 100644 index 0000000000..f8f0ee0331 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/monitor/prometheus/index.mdx @@ -0,0 +1,11 @@ +--- +description: Configure the open-source Prometheus monitoring and alerting toolkit bundled with Calico Cloud for component metrics, alerts, and persistent storage on connected clusters. +hide_table_of_contents: true +--- + +# Prometheus + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/operations/monitor/prometheus/support.mdx b/calico-cloud_versioned_docs/version-24-1/operations/monitor/prometheus/support.mdx new file mode 100644 index 0000000000..cde1d72c4e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/monitor/prometheus/support.mdx @@ -0,0 +1,19 @@ +--- +description: Reference for Prometheus support in Calico Cloud connected clusters covering the bundled operator-managed install and bring-your-own Prometheus deployment options. +--- + +# Prometheus support + +## Big picture + +$[prodname] uses the open-source [Prometheus monitoring and alerting toolkit](https://prometheus.io/docs/introduction/overview/). With these tools, you can view time-series metrics from $[prodname] components in the Prometheus and Grafana interfaces, or scrape the metrics for a BYO Prometheus deployment. + +## Install options + +- Use Prometheus operator managed by Tigera Operator + + You install the $[prodname] Prometheus operator and CRDs during $[prodname] installation. $[prodname] metrics and alerts are available in the web console. You configure alerts through Prometheus Alertmanager. + + If you want to specify your own Prometheus operator during installation for management by the Tigera Operator, the require operator version must be **v0.40.0 or higher**. Because $[prodname] creates Alertmanager and Prometheus CRs in the `tigera-prometheus` namespace, all you need to do is verify that your Prometheus operator is configured to manage Prometheus and Alertmanager instances in the `tigera-prometheus` namespace. + +- [Bring your own Prometheus](byo-prometheus.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/operations/usage-api.mdx b/calico-cloud_versioned_docs/version-24-1/operations/usage-api.mdx new file mode 100644 index 0000000000..74a01e8229 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/usage-api.mdx @@ -0,0 +1,43 @@ +--- +description: Create a read-only API key in the Calico Cloud web console and use it to call the Usage API to retrieve vCPU usage data programmatically. +--- + +import IconUser from '/img/icons/user-icon.svg'; + +# Access usage data programmatically + +You can create a read-only API key in the web console and use it to call the $[prodname] Usage API. +The API returns the same vCPU usage and managed cluster data shown on the [Usage Metrics](usage-metrics.mdx) page, so you can pull it into your own capacity planning, FinOps, and license-compliance workflows instead of exporting it by hand. +API keys are read-only and scoped to your organization. + +## Prerequisites + +* You are signed in to the web console as a user with the **Admin** and **Usage Metrics** roles, or with the **Owner** role. + +## Create an API key for usage metrics + +1. In the web console, click the user icon **> Settings**. +1. Under the **API Keys** tab, click **Generate new key**. +1. Enter a name, choose a validity period, and then click **Create**. +1. Copy or download the generated key and store it in a secure location. + + :::important + + The key is shown only once, at creation. You cannot retrieve it again after you close the dialog. + + ::: + +## Call the Usage API + +1. On the **API Keys** tab, locate your key and click **Actions > Generate commands**. +1. Select the command for the data you want: + * **Contracts** — your organization's contracts, including start and end dates, vCPU hours, package, and add-ons. + * **Summary** — per-project vCPU totals and usage over time, across all projects. + * **Managed clusters** — per-cluster usage rows, with optional project filtering and CSV output. +1. Copy the generated command. +1. In the command, replace `` with your API key, and set any query parameters, such as `startTime`, `endTime`, and `granularity`, for your query. +1. Run the command. A successful call returns an HTTP `200` response with your usage data in JSON. + +## Additional information + +* [Usage Metrics](usage-metrics.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/operations/usage-metrics.mdx b/calico-cloud_versioned_docs/version-24-1/operations/usage-metrics.mdx new file mode 100644 index 0000000000..f5b6391c92 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/operations/usage-metrics.mdx @@ -0,0 +1,59 @@ +--- +description: Reference for finding Calico Cloud Pro usage and billing metrics, calculated as vCPU hours across all connected clusters each month. +--- + +import IconUser from '/img/icons/user-icon.svg'; +import Screenshot from '/src/___new___/components/Screenshot'; + +# Calico Cloud Pro usage and billing + +To determine your monthly usage, we calculate the vCPU usage time in hours for your connected clusters. + +## Metering unit and standard pricing + +Usage is calculated monthly as a number of *vCPU hours*. +This includes the total vCPU hours for all your connected clusters. + +The standard pricing is US$0.025 per vCPU hour. + +There is no additional cost for log data ingestion. + +## View your usage metrics + +To view usage metrics, Calico Cloud Owners or users with Usage Metrics role can go to the web console, click the user icon , and then click **Usage Metrics**. + + + +## Subscription models + +| Subscription Model | Pricing | Usage | Additional Usage | +|---------------------|---------|--------|------------------| +| **Monthly** | US$0.025 per vCPU hour | Pay for actual usage each month | Billed at the same rate | +| **Annual** | Prepaid set of vCPU hours | Usage deducted from prepaid hours | Additional hours can be purchased; excess usage billed at US$0.025 per vCPU hour | + +
+ Example burndown table for an annual subscription + + The following table shows a the month-to-month usage for a cluster with an annual allocation of 12,000 vCPU hours. + + | Month | Starting Balance (vCPU hours) | Hours Consumed | Ending Balance (vCPU hours) | + |------------|-------------------------------|----------------|-----------------------------| + | January | 12,000 | 1,000 | 11,000 | + | February | 11,000 | 900 | 10,100 | + | March | 10,100 | 1,200 | 8,900 | + | April | 8,900 | 950 | 7,950 | + | May | 7,950 | 1,050 | 6,900 | + | June | 6,900 | 1,000 | 5,900 | + | July | 5,900 | 1,200 | 4,700 | + | August | 4,700 | 1,100 | 3,600 | + | September | 3,600 | 900 | 2,700 | + | October | 2,700 | 1,300 | 1,400 | + | November | 1,400 | 1,200 | 200 | + | December | 200 | 250 | -50 (overage applies) | + + :::note + If you're approaching your usage limit, you can purchase additional vCPU hours to avoid overage charges. + Otherwise, overage charges apply at the standard rate of US$0.025 per vCPU hour. + ::: + +
\ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/reference/api.mdx b/calico-cloud_versioned_docs/version-24-1/reference/api.mdx new file mode 100644 index 0000000000..4ec8fb41a6 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/api.mdx @@ -0,0 +1,12 @@ +--- +description: Calico Cloud Go client library reference for working with cluster resources programmatically against the Calico Cloud API in connected clusters. +--- + +# Tigera Client library + +$[prodname] provides and consumes a client library in Go that allows developers to work with $[prodname] resources. The +client library can be used to get, list, watch, create, delete and update custom resources, such as network policies much +like the [client-go project](https://github.com/kubernetes/client-go) does for native Kubernetes resources. + +To learn more about the Tigera client library and how to use it, see the Tigera API project [README](https://github.com/tigera/api/blob/master/README.md) or +the [github.com/tigera/api Go module page](https://pkg.go.dev/github.com/tigera/api). diff --git a/calico-cloud_versioned_docs/version-24-1/reference/architecture/data-path.mdx b/calico-cloud_versioned_docs/version-24-1/reference/architecture/data-path.mdx new file mode 100644 index 0000000000..d3d99e87c8 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/architecture/data-path.mdx @@ -0,0 +1,63 @@ +--- +description: Reference covering the Calico Cloud data path explaining how packets flow between workloads and to external destinations in connected clusters. +--- + +# 'The Calico Cloud data path: IP routing and iptables' + +One of $[prodname]’s key features is how packets flow between workloads in a +data center, or between a workload and the Internet, without additional +encapsulation. + +In the $[prodname] approach, IP packets to or from a workload are routed and +firewalled by the Linux routing table and iptables or eBPF infrastructure on the +workload’s host. For a workload that is sending packets, $[prodname] ensures +that the host is always returned as the next hop MAC address regardless +of whatever routing the workload itself might configure. For packets +addressed to a workload, the last IP hop is that from the destination +workload’s host to the workload itself. + +![Calico datapath](/img/calico-enterprise/calico-datapath.png) + +Suppose that IPv4 addresses for the workloads are allocated from a +datacenter-private subnet of 10.65/16, and that the hosts have IP +addresses from 172.18.203/24. If you look at the routing table on a host: + +```bash +route -n +``` + +You will see something like this: + +``` +Kernel IP routing table +Destination Gateway Genmask Flags Metric Ref Use Iface +0.0.0.0 172.18.203.1 0.0.0.0 UG 0 0 0 eth0 +10.65.0.0 0.0.0.0 255.255.0.0 U 0 0 0 ns-db03ab89-b4 +10.65.0.21 172.18.203.126 255.255.255.255 UGH 0 0 0 eth0 +10.65.0.22 172.18.203.129 255.255.255.255 UGH 0 0 0 eth0 +10.65.0.23 172.18.203.129 255.255.255.255 UGH 0 0 0 eth0 +10.65.0.24 0.0.0.0 255.255.255.255 UH 0 0 0 tapa429fb36-04 +172.18.203.0 0.0.0.0 255.255.255.0 U 0 0 0 eth0 +``` + +There is one workload on this host with IP address 10.65.0.24, and +accessible from the host via a TAP (or veth, etc.) interface named +tapa429fb36-04. Hence there is a direct route for 10.65.0.24, through +tapa429fb36-04. Other workloads, with the .21, .22 and .23 addresses, +are hosted on two other hosts (172.18.203.126 and .129), so the routes +for those workload addresses are via those hosts. + +The direct routes are set up by a $[prodname] agent named Felix when it is +asked to provision connectivity for a particular workload. A BGP client +(such as BIRD) then notices those and distributes them – perhaps via a +route reflector – to BGP clients running on other hosts, and hence the +indirect routes appear also. + +## Is that all? + +As far as the static data path is concerned, yes. It’s just a +combination of responding to workload ARP requests with the host MAC, IP +routing and iptables or eBPF. There’s a great deal more to $[prodname] in terms of +how the required routing and security information is managed, and for +handling dynamic things such as workload migration – but the basic data +path really is that simple. diff --git a/calico-cloud_versioned_docs/version-24-1/reference/architecture/design/index.mdx b/calico-cloud_versioned_docs/version-24-1/reference/architecture/design/index.mdx new file mode 100644 index 0000000000..154c4cbe06 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/architecture/design/index.mdx @@ -0,0 +1,11 @@ +--- +description: Network design reference for Calico Cloud connected clusters covering layer-2 and layer-3 interconnect fabric options for the underlay network. +hide_table_of_contents: true +--- + +# Network design + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/reference/architecture/design/l2-interconnect-fabric.mdx b/calico-cloud_versioned_docs/version-24-1/reference/architecture/design/l2-interconnect-fabric.mdx new file mode 100644 index 0000000000..fdb4b8312b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/architecture/design/l2-interconnect-fabric.mdx @@ -0,0 +1,238 @@ +--- +description: Reference for using Calico Cloud over a layer-2 Ethernet interconnect fabric covering BGP peering and broadcast domain considerations. +--- + +# Calico over Ethernet fabrics + +This is the first of a few _tech notes_ that I will be authoring that +will discuss some of the various interconnect fabric options in a $[prodname] +network. + +Any technology that is capable of transporting IP packets can be used as +the interconnect fabric in a $[prodname] network (the first person to test +and publish the results of using [IP over Avian Carrier](https://datatracker.ietf.org/doc/html/rfc1149) + as a transport for $[prodname] +will earn a very nice dinner on or with the core $[prodname] team). This +means that the standard tools used to transport IP, such as MPLS and +Ethernet can be used in a $[prodname] network. + +In this note, I'm going to focus on Ethernet as the interconnect +network. Talking to most at-scale cloud operators, they have converted +to IP fabrics, and as will cover in the next blog post that +infrastructure will work for $[prodname] as well. However, the concerns that +drove most of those operators to IP as the interconnection network in +their pods are largely ameliorated by Project Calico, allowing Ethernet +to be viably considered as a $[prodname] interconnect, even in large-scale +deployments. + +## Concerns over Ethernet at scale + +It has been acknowledged by the industry for years that, beyond a +certain size, classical Ethernet networks are unsuitable for production +deployment. Although there have been +[multiple](https://en.wikipedia.org/wiki/Provider_Backbone_Bridge_Traffic_Engineering) +[attempts](https://web.archive.org/web/20150923231827/https://www.cisco.com/web/about/ac123/ac147/archived_issues/ipj_14-3/143_trill.html) [to address](https://en.wikipedia.org/wiki/Virtual_Private_LAN_Service) +these issues, the scale-out networking community has, largely abandoned +Ethernet for anything other than providing physical point-to-point links +in the networking fabric. The principal reasons for Ethernet failures at +large scale are: + +1. Large numbers of _end points_ [^1]. Each switch in an Ethernet + network must learn the path to all Ethernet endpoints that are + connected to the Ethernet network. Learning this amount of state can + become a substantial task when we are talking about hundreds of + thousands of _end points_. +2. High rate of _churn_ or change in the network. With that many end + points, most of them being ephemeral (such as virtual machines or + containers), there is a large amount of _churn_ in the network. That + load of re-learning paths can be a substantial burden on the control + plane processor of most Ethernet switches. +3. High volumes of broadcast traffic. As each node on the Ethernet + network must use Broadcast packets to locate peers, and many use + broadcast for other purposes, the resultant packet replication to + each and every end point can lead to _broadcast storms_ in large + Ethernet networks, effectively consuming most, if not all resources + in the network and the attached end points. +4. Spanning tree. Spanning tree is the protocol used to keep an + Ethernet network from forming loops. The protocol was designed in + the era of smaller, simpler networks, and it has not aged well. As + the number of links and interconnects in an Ethernet network goes + up, many implementations of spanning tree become more _fragile_. + Unfortunately, when spanning tree fails in an Ethernet network, the + effect is a catastrophic loop or partition (or both) in the network, + and, in most cases, difficult to troubleshoot or resolve. + +While many of these issues are crippling at _VM scale_ (tens of +thousands of end points that live for hours, days, weeks), they will be +absolutely lethal at _container scale_ (hundreds of thousands of end +points that live for seconds, minutes, days). + +If you weren't ready to turn off your Ethernet data center network +before this, I bet you are now. Before you do, however, let's look at +how Project Calico can mitigate these issues, even in very large +deployments. + +## How does $[prodname] tame the Ethernet daemons? + +First, let's look at how $[prodname] uses an Ethernet interconnect fabric. +It's important to remember that an Ethernet network _sees_ nothing on +the other side of an attached IP router, the Ethernet network just +_sees_ the router itself. This is why Ethernet switches can be used at +Internet peering points, where large fractions of Internet traffic is +exchanged. The switches only see the routers from the various ISPs, not +those ISPs' customers' nodes. We leverage the same effect in $[prodname]. + +To take the issues outlined above, let's revisit them in a $[prodname] +context. + +1. Large numbers of end points. In a $[prodname] network, the Ethernet + interconnect fabric only sees the routers/compute servers, not the + end point. In a standard cloud model, where there is tens of VMs per + server (or hundreds of containers), this reduces the number of nodes + that the Ethernet sees (and has to learn) by one to two orders + of magnitude. Even in very large pods (say twenty thousand servers), + the Ethernet network would still only see a few tens of thousands of + end points. Well within the scale of any competent data center + Ethernet top of rack (ToR) switch. +2. High rate of _churn_. In a classical Ethernet data center fabric, + there is a _churn_ event each time an end point is created, + destroyed, or moved. In a large data center, with hundreds of + thousands of endpoints, this _churn_ could run into tens of events + per second, every second of the day, with peaks easily in the + hundreds or thousands of events per second. In a $[prodname] network, + however, the _churn_ is very low. The only event that would lead to + _churn_ in a $[prodname] network's Ethernet fabric would be the addition + or loss of a compute server, switch, or physical connection. In a + twenty thousand server pod, even with a 5% daily failure rate (a few + orders of magnitude more than what is normally experienced), there + would only be two thousand events per **day**. Any switch that can + not handle that volume of change in the network should not be used + for any application. +3. High volume of broadcast traffic. Since the first (and last) hop for + any traffic in a $[prodname] network is an IP hop, and IP hops terminate + broadcast traffic, there is no endpoint broadcast network in the + Ethernet fabric, period. In fact, the only broadcast traffic that + should be seen in the Ethernet fabric is the ARPs of the compute + servers locating each other. If the traffic pattern is fairly + consistent, the steady-state ARP rate should be almost zero. Even in + a pathological case, the ARP rate should be well within normal + accepted boundaries. +4. Spanning tree. Depending on the architecture chosen for the Ethernet + fabric, it may even be possible to turn off spanning tree. However, + even if it is left on, due to the reduction in node count, and + reduction in churn, most competent spanning tree implementations + should be able to handle the load without stress. + +With these considerations in mind, it should be evident that an Ethernet +connection fabric in $[prodname] is not only possible, it is practical and +should be seriously considered as the interconnect fabric for a $[prodname] +network. + +As mentioned in the IP fabric post, an IP fabric is also quite feasible +for $[prodname], but there are more considerations that must be taken into +account. The Ethernet fabric option has fewer architectural +considerations in its design. + +## A brief note about Ethernet topology + +As mentioned elsewhere in the $[prodname] documentation, since $[prodname] can use +most of the standard IP tooling, some interesting options regarding +fabric topology become possible. + +We assume that an Ethernet fabric for $[prodname] would most likely be +constructed as a _leaf/spine_ architecture. Other options are possible, +but the _leaf/spine_ is the predominant architectural model in use in +scale-out infrastructure today. + +Since $[prodname] is an IP routed fabric, a $[prodname] network can use +[ECMP](https://en.wikipedia.org/wiki/Equal-cost_multi-path_routing) to +distribute traffic across multiple links (instead of using Ethernet +techniques such as MLAG). By leveraging ECMP load balancing on the +$[prodname] compute servers, it is possible to build the fabric out of +multiple _independent_ leaf/spine planes using no technologies other +than IP routing in the $[prodname] nodes, and basic Ethernet switching in the +interconnect fabric. These planes would operate completely independently +and could be designed such that they would not share a fault domain. +This would allow for the catastrophic failure of one (or more) plane(s) +of Ethernet interconnect fabric without the loss of the pod (the failure +would just decrease the amount of interconnect bandwidth in the pod). +This is a gentler failure mode than the pod-wide IP or Ethernet failure +that is possible with today's designs. + +A more in-depth discussion is possible, so if you'd like, please make a +request, and I will put up a post or white paper. In the meantime, it +may be interesting to venture over to Facebook's [blog post](https://engineering.fb.com/2014/11/14/production-engineering/introducing-data-center-fabric-the-next-generation-facebook-data-center-network/) + +on their fabric approach. A quick picture to visualize the idea is shown +below. + +![A diagram showing the Ethernet spine planes. Each color represents a distinct Ethernet network, transporting a unique IP network.](/img/calico-enterprise/l2-spine-planes.png) + +I am not showing the end points in this diagram, and the end points +would be unaware of anything in the fabric (as noted above). + +In the particular case of this diagram, each ToR is segmented into four +logical switches (possibly by using 'port VLANs'), [^2] and each compute +server has a connection to each of those logical switches. We will +identify those logical switches by their color. Each ToR would then have +a blue, green, orange, and red logical switch. Those 'colors' would be +members of a given _plane_, so there would be a blue plane, a green +plane, an orange plane, and a red plane. Each plane would have a +dedicated spine switch. and each ToR in a given spine would be connected +to its spine, and only its spine. + +Each plane would constitute an IP network, so the blue plane would be +2001:db8:1000::/36, the green would be 2001:db8:2000::/36, and the +orange and red planes would be 2001:db8:3000::/36 and 2001:db8:4000::/36 +respectively. [^3] + +Each IP network (plane) requires its own BGP route reflectors. Those +route reflectors need to be peered with each other within the plane, but +the route reflectors in each plane do not need to be peered with one +another. Therefore, a fabric of four planes would have four route +reflector meshes. Each compute server, border router, _etc._ would need +to be a route reflector client of at least one route reflector in each +plane, and very preferably two or more in each plane. + +A diagram that visualizes the route reflector environment can be found +below. + +![A diagram showing the route reflector topology in the l2 spine plane architecture. The dashed diamonds are the route reflectors, with one or more per L2 spine plane. All compute servers are peered to all route reflectors, and all the route reflectors in a given plane are also meshed. However, the route reflectors in each spine plane are not meshed together (*e.g.* the *blue* route reflectors are not peered or meshed with the *red* route reflectors. The route reflectors themselves could be daemons running on the actual compute servers or on other dedicated or networking hardware.](/img/calico-enterprise/l2-rr-spine-planes.png) + +These route reflectors could be dedicated hardware connected to the +spine switches (or the spine switches themselves), or physical or +virtual route reflectors connected to the necessary logical leaf +switches (blue, green, orange, and red). That may be a route reflector +running on a compute server and connected directly to the correct plane +link, and not routed through the vRouter, to avoid the chicken and egg +problem that would occur if the route reflector were "behind" the $[prodname] +network. + +Other physical and logical configurations and counts are, of course, +possible, this is just an example. + +The logical configuration would then have each compute server would have +an address on each plane's subnet, and announce its end points on each +subnet. If ECMP is then turned on, the compute servers would distribute +the load across all planes. + +If a plane were to fail (say due to a spanning tree failure), then only +that one plane would fail. The remaining planes would stay running. + +[^1]: + In this document (and in all $[prodname] documents) we tend to use the + terms _end point_ to refer to a virtual machine, container, + appliance, bare metal server, or any other entity that is connected + to a $[prodname] network. If we are referring to a specific type of end + point, we will call that out (such as referring to the behavior of + VMs as distinct from containers). + +[^2]: + We are using logical switches in this example. Physical ToRs could + also be used, or a mix of the two (say 2 logical switches hosted on + each physical switch). + +[^3]: + We use IPv6 here purely as an example. IPv4 would be configured + similarly. I welcome your questions, either here on the blog, or via + the Project Calico mailing list. diff --git a/calico-cloud_versioned_docs/version-24-1/reference/architecture/design/l3-interconnect-fabric.mdx b/calico-cloud_versioned_docs/version-24-1/reference/architecture/design/l3-interconnect-fabric.mdx new file mode 100644 index 0000000000..1db0910c0a --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/architecture/design/l3-interconnect-fabric.mdx @@ -0,0 +1,563 @@ +--- +description: Reference for using Calico Cloud over a layer-3 IP interconnect fabric covering BGP topology choices and route propagation. +--- + +# Calico over IP fabrics + +$[prodname] provides an end-to-end IP network that interconnects the +endpoints [^1] in a scale-out or cloud environment. To do that, it needs +an _interconnect fabric_ to provide the physical networking layer on +which $[prodname] operates [^2]. + +While $[prodname] is designed to work with any underlying interconnect fabric +that can support IP traffic, the fabric that has the least +considerations attached to its implementation is an Ethernet fabric as +discussed in our earlier [technical note](l2-interconnect-fabric.mdx). + +In most cases, the Ethernet fabric is the appropriate choice, but there +are infrastructures where L3 (an IP fabric) has already been deployed, +or will be deployed, and it makes sense for $[prodname] to operate in those +environments. + +However, since $[prodname] is, itself, a routed infrastructure, there are +more engineering, architecture, and operations considerations that have +to be weighed when running $[prodname] with an IP routed interconnection +fabric. We will briefly outline those in the rest of this post. That +said, $[prodname] operates equally well with Ethernet or IP interconnect +fabrics. + +## Background + +### Basic $[prodname] architecture overview +{/*TODO-XREFS-CC +A description of the $[prodname] architecture can be found in our +[architectural overview](/reference/architecture/overview). +However, a brief discussion of the routing and data paths is useful for +the discussion. +*/} + +In a $[prodname] network, each compute server acts as a router for all of the +endpoints that are hosted on that compute server. We call that function +a vRouter. The data path is provided by the Linux kernel, the control +plane by a BGP protocol server, and management plane by $[prodname]'s +on-server agent, _Felix_. + +Each endpoint can only communicate through its local vRouter, and the +first and last _hop_ in any $[prodname] packet flow is an IP router hop +through a vRouter. Each vRouter announces all of the endpoints it is +attached to all the other vRouters and other routers on the +infrastructure fabric, using BGP, usually with BGP route reflectors to +increase scale. A discussion of why we use BGP can be found in the [Why BGP?](https://www.projectcalico.org/why-bgp/) + blog post. + +Access control lists (ACLs) enforce security (and other) policy as +directed by whatever cloud orchestrator is in use. There are other +components in the $[prodname] architecture, but they are irrelevant to the +interconnect network fabric discussion. + +### Overview of current common IP scale-out fabric architectures + +There are two approaches to building an IP fabric for a scale-out +infrastructure. However, all of them, to date, have assumed that the +edge router in the infrastructure is the top of rack (TOR) switch. In +the $[prodname] model, that function is pushed to the compute server itself. + +The two approaches are outlined below, in this technical note, we will +cover the second option, as it is more common in the scale-out world. If +there is interest in the first approach, please contact Project Calico, +and we can discuss, and if there is enough interest, maybe we will do +another technical note on that approach. If you know of other approaches +in use, we would be happy to host a guest technical note. + +1. The routing infrastructure is based on some form of IGP. Due to the + limitations in scale of IGP networks (see the [why BGP post](https://www.projectcalico.org/why-bgp/) + for discussion of this + topic), the Project Calico team does not believe that using an IGP + to distribute endpoint reachability information will adequately + scale in a $[prodname] environment. However, it is possible to use a + combination of IGP and BGP in the interconnect fabric, where an IGP + communicates the path to the _next-hop_ router (in $[prodname], this is + often the destination compute server) and BGP is used to distribute + the actual next-hop for a given endpoint. This is a valid model, + and, in fact is the most common approach in a widely distributed IP + network (say a carrier's backbone network). The design of these + networks is somewhat complex though, and will not be addressed + further in this technical note. [^3] +2. The other model, and the one that this note concerns itself with, is + one where the routing infrastructure is based entirely on BGP. In + this model, the IP network is "tight enough" or has a small enough + diameter that BGP can be used to distribute endpoint routes, and the + paths to the next-hops for those routes is known to all of the + routers in the network (in a $[prodname] network this includes the + compute servers). This is the network model that this note + will address. + +### BGP-only interconnect fabrics + +There are multiple methods to build a BGP-only interconnect fabric. We +will focus on three models, each with two widely viable variations. +There are other options, and we will briefly touch on why we didn't +include some of them in the [Other Options appendix](#other-options). + +The two methods are: + +1. A BGP fabric where each of the TOR switches (and their subsidiary + compute servers) are a unique [Autonomous System (AS)]() + and they are interconnected via either an Ethernet switching plane + provided by the spine switches in a + [leaf/spine](http://bradhedlund.com/2012/10/24/video-a-basic-introduction-to-the-leafspine-data-center-networking-fabric-design/) + architecture, or via a set of spine switches, each of which is also + a unique AS. We'll refer to this as the _AS per rack_ model. This + model is detailed in [IETF RFC 7938](https://datatracker.ietf.org/doc/html/rfc7938). +2. A BGP fabric where each of the compute servers is a unique AS, and + the TOR switches make up a transit AS. We'll refer to this as the + _AS per server_ model. + +Each of these models can either have an Ethernet or IP spine. In the +case of an Ethernet spine, each spine switch provides an isolated +Ethernet connection _plane_ as in the $[prodname] Ethernet interconnect +fabric model and each TOR switch is connected to each spine switch. + +Another model is where each spine switch is a unique AS, and each TOR +switch BGP peers with each spine switch. In both cases, the TOR switches +use ECMP to load-balance traffic between all available spine switches. + +### Some BGP network design considerations + +Contrary to popular opinion, BGP is actually a fairly simple protocol. +For example, the BGP configuration on a $[prodname] compute server is +approximately sixty lines long, not counting comments. The perceived +complexity is due to the things that you can _do_ with BGP. Many uses of +BGP involve complex policy rules, where the behavior of BGP can be +modified to meet technical (or business, financial, political, _etc._) +requirements. A default $[prodname] network does not venture into those +areas, [^4] and therefore is fairly straight forward. + +That said, there are a few design rules for BGP that need to be kept in +mind when designing an IP fabric that will interconnect nodes in a +$[prodname] network. These BGP design requirements _can_ be worked around, if +necessary, but doing so takes the designer out of the standard BGP +_envelope_ and should only be done by an implementer who is _very_ +comfortable with advanced BGP design. + +These considerations are: + +
+
AS continuity
+
or AS puddling Any router in an AS must be able to communicate + with any other router in that same AS without transiting another AS.
+
Next hop behavior
+
By default BGP routers do not change the next hop of a route if it + is peering with another router in its same AS. The inverse is also + true, a BGP router will set itself as the next hop of a route if + it is peering with a router in another AS.
+
Route reflection
+
All BGP routers in a given AS must peer with all the other routers + in that AS. This is referred to a complete BGP mesh. This can + become problematic as the number of routers in the AS scales up. The + use of route reflectors reduce the need for the complete BGP mesh. + However, route reflectors also have scaling considerations.
+
Endpoints
+
In a $[prodname] network, each endpoint is a route. Hardware networking + platforms are constrained by the number of routes they can learn. + This is usually in range of 10,000's or 100,000's of routes. Route + aggregation can help, but that is usually dependent on the + capabilities of the scheduler used by the orchestration software.
+
+ +A deeper discussion of these considerations can be found in the IP +Fabric Design Considerations\_ appendix. + +The designs discussed below address these considerations. + +### The _AS Per Rack_ model + +This model is the closest to the model suggested by [IETF RFC 7938](https://datatracker.ietf.org/doc/html/rfc7938). + +As mentioned earlier, there are two versions of this model, one with an +set of Ethernet planes interconnecting the ToR switches, and the other +where the core planes are also routers. The following diagrams may be +useful for the discussion. + +![Diagram showing the AS per rack model with ToR switches meshed via Ethernet switching planes at the spine layer](/img/calico-enterprise/l3-fabric-diagrams-as-rack-l2-spine.png) + +The diagram above shows the _AS per rack model_ where the ToR switches are +physically meshed via a set of Ethernet switching planes. + +![Diagram showing the AS per rack model with ToR switches meshed via discrete BGP spine routers, each in their own AS](/img/calico-enterprise/l3-fabric-diagrams-as-rack-l3-spine.png) + +The diagram above shows the _AS per rack model_ where the ToR switches are +physically meshed via a set of discrete BGP spine routers, each in +their own AS. + +In this approach, every ToR-ToR or ToR-Spine (in the case of an AS per +spine) link is an eBGP peering which means that there is no +route-reflection possible (using standard BGP route reflectors) _north_ +of the ToR switches. + +If the L2 spine option is used, the result of this is that each ToR must +either peer with every other ToR switch in the cluster (which could be +hundreds of peers). + +If the AS per spine option is used, then each ToR only has to peer with +each spine (there are usually somewhere between two and sixteen spine +switches in a pod). However, the spine switches must peer with all ToR +switches (again, that would be hundreds, but most spine switches have +more control plane capacity than the average ToR, so this might be more +scalable in many circumstances). + +Within the rack, the configuration is the same for both variants, and is +somewhat different than the configuration north of the ToR. + +Every router within the rack, which, in the case of $[prodname] is every +{/* vale Vale.Repetition = NO */} +compute server, shares the same AS as the ToR that they are connected +{/* vale Vale.Repetition = YES */} +to. That connection is in the form of an Ethernet switching layer. Each +router in the rack must be directly connected to enable the AS to remain +contiguous. The ToR's _router_ function is then connected to that +Ethernet switching layer as well. The actual configuration of this is +dependent on the ToR in use, but usually it means that the ports that +are connected to the compute servers are treated as _subnet_ or +_segment_ ports, and then the ToR's _router_ function has a single +interface into that subnet. + +This configuration allows each compute server to connect to each other +compute server in the rack without going through the ToR router, but it +will, of course, go through the ToR switching function. The compute +servers and the ToR router could all be directly meshed, or a route +reflector could be used within the rack, either hosted on the ToR +itself, or as a virtual function hosted on one or more compute servers +within the rack. + +The ToR, as the eBGP router redistributes all of the routes from other +ToRs as well as routes external to the data center to the compute +servers that are in its AS, and announces all of the routes from within +the AS (rack) to the other ToRs and the larger world. This means that +each compute server will see the ToR as the next hop for all external +routes, and the individual compute servers are the next hop for all +routes internal to the rack. + +### The _AS per Compute Server_ model + +This model takes the concept of an AS per rack to its logical +conclusion. In the earlier referenced [IETF RFC 7938](https://datatracker.ietf.org/doc/html/rfc7938) +the assumption in the overall model is that the ToR is first tier +aggregating and routing element. In $[prodname], the ToR, if it is an L3 +router, is actually the second tier. Remember, in $[prodname], the compute +server is always the first/last router for an endpoint, and is also the +first/last point of aggregation. + +Therefore, if we follow the architecture of the draft, the compute +server, not the ToR should be the AS boundary. The differences can be +seen in the following two diagrams. + +![Diagram showing the AS per compute server model with ToR switches meshed via Ethernet switching planes at the spine layer](/img/calico-enterprise/l3-fabric-diagrams-as-server-l2-spine.png) + +The diagram above shows the _AS per compute server model_ where the ToR +switches are physically meshed via a set of Ethernet switching planes. + +![Diagram showing the AS per compute server model with ToR switches connected to independent routing planes at the spine layer](/img/calico-enterprise/l3-fabric-diagrams-as-server-l3-spine.png) + +The diagram above shows the _AS per compute server model_ where the ToR +switches are physically connected to a set of independent routing +planes. + +As can be seen in these diagrams, there are still the same two variants +as in the _AS per rack_ model, one where the spine switches provide a +set of independent Ethernet planes to interconnect the ToR switches, and +the other where that is done by a set of independent routers. + +The real difference in this model, is that the compute servers as well +as the ToR switches are all independent autonomous systems. To make this +work at scale, the use of four byte AS numbers as discussed in +[RFC 4893](http://www.faqs.org/rfcs/rfc4893.html 'RFC 4893'). Without +using four byte AS numbering, the total number of ToRs and compute +servers in a $[prodname] fabric would be limited to the approximately five +thousand available private AS [^5] numbers. If four byte AS numbers are +used, there are approximately ninety-two million private AS numbers +available. This should be sufficient for any given $[prodname] fabric. + +The other difference in this model _vs._ the AS per rack model, is that +there are no route reflectors used, as all BGP peerings are eBGP. In +this case, each compute server in a given rack peers with its ToR switch +which is also acting as an eBGP router. For two servers within the same +rack to communicate, they will be routed through the ToR. Therefore, +each server will have one peering to each ToR it is connected to, and +each ToR will have a peering with each compute server that it is +connected to (normally, all the compute servers in the rack). + +The inter-ToR connectivity considerations are the same in scale and +scope as in the AS per rack model. + +### The _Downward Default_ model + +The final model is a bit different. Whereas, in the previous models, all +of the routers in the infrastructure carry full routing tables, and +leave their AS paths intact, this model [^6] removes the AS numbers at +each stage of the routing path. This is to prevent routes from other +nodes in the network from not being installed due to it coming from the +_local_ AS (since they share the source and dest of the route share the +same AS). + +The following diagram will show the AS relationships in this model. + +![Diagram showing the downward default model where all Calico nodes share one AS and all ToR switches share another, with spine routers announcing default routes downward](/img/calico-enterprise/l3-fabric-downward-default.png) + +In the diagram above, we are showing that all $[prodname] nodes share the same +AS number, as do all ToR switches. However, those ASs are different +(_A1_ is not the same network as _A2_, even though the both share the +same AS number _A_ ). + +While the use of a single AS for all ToR switches, and another for all +compute servers simplifies deployment (standardized configuration), the +real benefit comes in the offloading of the routing tables in the ToR +switches. + +In this model, each router announces all of its routes to its upstream +peer (the $[prodname] routers to their ToR, the ToRs to the spine switches). +However, in return, the upstream router only announces a default route. +In this case, a given $[prodname] router only has routes for the endpoints +that are locally hosted on it, as well as the default from the ToR. +Since the ToR is the only route for the $[prodname] network the rest of the +network, this matches reality. The same happens between the ToR switches +and the spine. This means that the ToR only has to install the routes +that are for endpoints that are hosted on its downstream $[prodname] nodes. +Even if we were to host 200 endpoints per $[prodname] node, and stuff 80 +$[prodname] nodes in each rack, that would still limit the routing table on +the ToR to a maximum of 16,000 entries (well within the capabilities of +even the most modest of switches). + +Since the default is originated by the Spine (originally) there is no +chance for a downward announced route to originate from the recipient's +AS, preventing the _AS puddling_ problem. + +There is one (minor) drawback to this model, in that all traffic that is +destined for an invalid destination (the destination IP does not exist) +will be forwarded to the spine switches before they are dropped. + +It should also be noted that the spine switches do need to carry all of +the $[prodname] network routes, just as they do in the routed spines in the +previous examples. In short, this model imposes no more load on the +spines than they already would have, and substantially reduces the +amount of routing table space used on the ToR switches. It also reduces +the number of routes in the $[prodname] nodes, but, as we have discussed +before, that is not a concern in most deployments as the amount of +memory consumed by a full routing table in $[prodname] is a fraction of the +total memory available on a modern compute server. + +## Recommendation + +The Project Calico team recommends the use of the [AS per rack](#the-as-per-rack-model) model if +the resultant routing table size can be accommodated by the ToR and +spine switches, remembering to account for projected growth. + +If there is concern about the route table size in the ToR switches, the +team recommends the [Downward Default](#the-downward-default-model) model. + +If there are concerns about both the spine and ToR switch route table +capacity, or there is a desire to run a very simple L2 fabric to connect +the $[prodname] nodes, then the user should consider the Ethernet fabric as +detailed in [this post](l2-interconnect-fabric.mdx). + +If a $[prodname] user is interested in the AS per compute server, the Project +Calico team would be very interested in discussing the deployment of +that model. + +## Appendix + +### Other Options + +The way the physical and logical connectivity is laid out in this note, +and the [Ethernet fabric note](l2-interconnect-fabric.mdx), +The next hop router for a given route is always directly connected to +the router receiving that route. This makes the need for another +protocol to distribute the next hop routes unnecessary. + +However, in many (or most) WAN BGP networks, the routers within a given +AS may not be directly adjacent. Therefore, a router may receive a route +with a next hop address that it is not directly adjacent to. In those +cases, an IGP, such as OSPF or IS-IS, is used by the routers within a +given AS to determine the path to the BGP next hop route. + +There may be $[prodname] architectures where there are similar models where +the routers within a given AS are not directly adjacent. In those +models, the use of an IGP in $[prodname] may be warranted. The configuration +of those protocols are, however, beyond the scope of this technical +note. + +### IP Fabric Design Considerations + +#### AS puddling + +The first consideration is that an AS must be kept contiguous. This +means that any two nodes in a given AS must be able to communicate +without traversing any other AS. If this rule is not observed, the +effect is often referred to as _AS puddling_ and the network will _not_ +function correctly. + +A corollary of that rule is that any two administrative regions that +share the same AS number, are in the same AS, even if that was not the +desire of the designer. BGP has no way of identifying if an AS is local +or foreign other than the AS number. Therefore re-use of an AS number +for two _networks_ that are not directly connected, but only connected +through another _network_ or AS number will not work without a lot of +policy changes to the BGP routers. + +{/* vale Vale.Repetition = NO */} +Another corollary of that rule is that a BGP router will not propagate a +route to a peer if the route has an AS in its path that is the same AS +{/* vale Vale.Repetition = YES */} +as the peer. This prevents loops from forming in the network. The effect +of this prevents two routers in the same AS from transiting another +router (either in that AS or not). + +#### Next hop behavior + +Another consideration is based on the differences between iBGP and eBGP. +BGP operates in two modes, if two routers are BGP peers, but share the +same AS number, then they are considered to be in an _internal_ BGP (or +iBGP) peering relationship. If they are members of different AS's, then +they are in an _external_ or eBGP relationship. + +BGP's original design model was that all BGP routers within a given AS +would know how to get to one another (via static routes, IGP [^7] +routing protocols, or the like), and that routers in different ASs would +not know how to reach one another unless they were directly connected. + +Based on that design point, routers in an iBGP peering relationship +assume that they do not transit traffic for other iBGP routers in a +given AS (i.e. A can communicate with C, and therefore will not need to +route through B), and therefore, do not change the _next hop_ attribute +in BGP [^8]. + +A router with an eBGP peering, on the other hand, assumes that its eBGP +peer will not know how to reach the next hop route, and then will +substitute its own address in the next hop field. This is often referred +to as _next hop self_. + +In the $[prodname] [Ethernet fabric](l2-interconnect-fabric.mdx) +model, all of the compute servers (the routers in a $[prodname] network) are +directly connected over one or more Ethernet network(s) and therefore +are directly reachable. In this case, a router in the $[prodname] network +does not need to set _next hop self_ within the $[prodname] fabric. + +The models we present in this technical note insure that all routes that +may traverse a non-$[prodname] router are eBGP routes, and therefore _next +hop self_ is automatically set correctly. If a deployment of $[prodname] in +an IP interconnect fabric does not satisfy that constraint, then _next +hop self_ must be appropriately configured. + +#### Route reflection + +As mentioned above, BGP expects that all of the iBGP routers in a +network can see (and speak) directly to one another, this is referred to +as a _BGP full mesh_. In small networks this is not a problem, but it +does become interesting as the number of routers increases. For example, +if you have 99 BGP routers in an AS and wish to add one more, you would +have to configure the peering to that new router on each of the 99 +existing routers. Not only is this a problem at configuration time, it +means that each router is maintaining 100 protocol adjacencies, which +can start being a drain on constrained resources in a router. While this +might be _interesting_ at 100 routers, it becomes an impossible task +with 1000's or 10,000's of routers (the potential size of a $[prodname] +network). + +Conveniently, large scale/Internet scale networks solved this problem +almost 20 years ago by deploying BGP route reflection as described in +[RFC 1966](http://www.faqs.org/rfcs/rfc1966.html 'RFC 1966'). This is a +technique supported by almost all BGP routers today. In a large network, +a number of route reflectors [^9] are evenly distributed and each iBGP +router is _peered_ with one or more route reflectors (usually 2 or 3). +Each route reflector can handle 10's or 100's of route reflector clients +(in $[prodname]'s case, the compute server), depending on the route reflector +being used. Those route reflectors are, in turn, peered with each other. +This means that there are an order of magnitude less route reflectors +that need to be completely meshed, and each route reflector client is +only configured to peer to 2 or 3 route reflectors. This is much easier +to manage. + +Other route reflector architectures are possible, but those are beyond +the scope of this document. + +#### Endpoints + +The final consideration is the number of endpoints in a $[prodname] network. +In the [Ethernet fabric](l2-interconnect-fabric.mdx) +case the number of endpoints is not constrained by the interconnect +fabric, as the interconnect fabric does not _see_ the actual endpoints, +it only _sees_ the actual vRouters, or compute servers. This is not the +case in an IP fabric, however. IP networks forward by using the +destination IP address in the packet, which, in $[prodname]'s case, is the +destination endpoint. That means that the IP fabric nodes (ToR switches +and/or spine switches, for example) must know the routes to each +endpoint in the network. They learn this by participating as route +reflector clients in the BGP mesh, just as the $[prodname] vRouter/compute +server does. + +However, unlike a compute server which has a relatively unconstrained +amount of memory, a physical switch is either memory constrained, or +quite expensive. This means that the physical switch has a limit on how +many _routes_ it can handle. The current industry standard for modern +commodity switches is in the range of 128,000 routes. This means that, +without other routing _tricks_, such as aggregation, a $[prodname] +installation that uses an IP fabric will be limited to the routing table +size of its constituent network hardware, with a reasonable upper limit +today of 128,000 endpoints. + +[^1]: + In $[prodname]'s terminology, an endpoint is an IP address and + interface. It could refer to a VM, a container, or even a process + bound to an IP address running on a bare metal server. + +[^2]: + This interconnect fabric provides the connectivity between the + $[prodname] (v)Router (in almost all cases, the compute servers) nodes, + as well as any other elements in the fabric (_e.g._ bare metal + servers, border routers, and appliances). + +[^3]: + If there is interest in a discussion of this approach, please let + us know. The Project Calico team could either arrange a discussion, + or if there was enough interest, publish a follow-up tech note. + +[^4]: + However those tools are available if a given $[prodname] instance needs + to utilize those policy constructs. + +[^5]: + The two byte AS space reserves approximately the last five + thousand AS numbers for private use. There is no technical reason + why other AS numbers could not be used. However the re-use of global + scope AS numbers within a private infrastructure is strongly + discouraged. The chance for routing system failure or incorrect + routing is substantial, and not restricted to the entity that is + doing the reuse. + +[^6]: + We first saw this design in a customer's lab, and thought it + innovative enough to share (we asked them first, of course). Similar + _AS Path Stripping_ approaches are used in ISP networks, however. + +[^7]: + An Interior Gateway Protocol is a local routing protocol that does + not cross an AS boundary. The primary IGPs in use today are OSPF and + IS-IS. While complex iBGP networks still use IGP routing protocols, + a data center is normally a fairly simple network, even if it has + many routers in it. Therefore, in the data center case, the use of + an IGP can often be disposed of. + +[^8]: + A Next hop is an attribute of a route announced by a routing + protocol. In simple terms a route is defined by a _target_, or the + destination that is to be reached, and a _next hop_, which is the + next router in the path to reach that target. There are many other + characteristics in a route, but those are well beyond the scope of + this post. + +[^9]: + A route reflector may be a physical router, a software appliance, + or simply a BGP daemon. It only processes routing messages, and does + not pass actual data plane traffic. However, some route reflectors + are co-resident on regular routers that do pass data plane traffic. + While they may sit on one platform, the functions are distinct. diff --git a/calico-cloud_versioned_docs/version-24-1/reference/architecture/index.mdx b/calico-cloud_versioned_docs/version-24-1/reference/architecture/index.mdx new file mode 100644 index 0000000000..a6838e6e4d --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/architecture/index.mdx @@ -0,0 +1,11 @@ +--- +description: Reference content for Calico Cloud architecture including connected cluster diagram, network design, and the data path between workloads. +hide_table_of_contents: true +--- + +# Architecture + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/reference/attribution.mdx b/calico-cloud_versioned_docs/version-24-1/reference/attribution.mdx new file mode 100644 index 0000000000..c66b171c27 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/attribution.mdx @@ -0,0 +1,3394 @@ +--- +description: Third-party license attribution report listing open-source components shipped with Calico Cloud. +--- + +# Attribution + +## $[prodname] attribution report + +24 Feb 2021 + +$[prodname] incorporates various open source software. The following open source components and their respective licenses used in the product are provided for your informational purposes. +In the table below, you can look at the details of each project and license associated with it. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ProjectArtifact NameLICENSE
abbrevabbrev-1.1.1.tgzISC
algoliasearchalgoliasearch.umd-4.2.0.jsMIT
ansi-regexansi-regex-2.1.1.tgzMIT
ansi-stylesansi-styles-3.2.1.tgzMIT
@types/anymatchanymatch-1.3.1.tgzMIT
app-policyapp-policy-v3.16.6Tigera Proprietary
argparseargparse-1.0.9.tgzMIT
arr-diffarr-diff-4.0.0.tgzMIT
arr-flattenarr-flatten-1.1.0.tgzMIT
arr-unionarr-union-3.1.0.tgzMIT
array-uniquearray-unique-0.3.2.tgzMIT
asapasap-2.0.6.tgzMIT
assign-symbolsassign-symbols-1.0.0.tgzMIT
atobatob-2.1.2.tgzApache 2.0
atomicwritesatomicwrites-1.4.0-py2.py3-none-any.whlMIT
attrsattrs-20.3.0-py2.py3-none-any.whlMIT
avsdf-baseavsdf-base-1.0.0.tgzMIT
babel-standalonebabel-6.26.0.min.jsMIT
babel-runtimebabel-runtime-6.26.0.tgzMIT
backports.functools-lru-cachebackports.functools_lru_cache-1.6.1-py2.py3-none-any.whlMIT
balanced-matchbalanced-match-0.4.2.tgzMIT
balanced-matchbalanced-match-1.0.0.tgzMIT
basebase-0.11.2.tgzMIT
beautifulsoup4beautifulsoup4-4.9.3-py2-none-any.whlMIT
big.jsbig.js-5.2.2.tgzMIT
boolbaseboolbase-1.0.0.tgzISC
twitter-bootstrapbootstrap-3.3.7.min.jsMIT
bootstrapbootstrap-3.4.1.tgzMIT
brace-expansionbrace-expansion-1.1.11.tgzMIT
bracesbraces-2.3.2.tgzMIT
bs4bs4-0.0.1.tar.gzMIT
buffer-frombuffer-from-1.1.1.tgzMIT
cache-basecache-base-1.0.1.tgzMIT
cachetoolscachetools-3.1.1-py2.py3-none-any.whlMIT
projectcalicocalico-v3.17.1Apache 2.0
projectcalicocalicoctl-v3.17.1Apache 2.0
camel-casecamel-case-4.1.1.tgzMIT
certificertifi-2020.12.5-py2.py3-none-any.whlMozilla 2.0
chain-functionchain-function-1.0.0.tgzMIT
chalkchalk-2.4.2.tgzMIT
chardetchardet-3.0.4-py2.py3-none-any.whlLGPL 3.0
chardetchardet-4.0.0-py2.py3-none-any.whlLGPL 3.0
chevrotainchevrotain-6.5.0.tgzApache 2.0
cidr-regexcidr-regex-2.0.10.tgzBSD 2
class-transformerclass-transformer-0.3.1.tgzMIT
class-utilsclass-utils-0.3.6.tgzMIT
class-validatorclass-validator-0.9.1.tgzMIT
classnamesclassnames-2.2.5.tgzMIT
classnamesclassnames-2.2.6.tgzMIT
clean-cssclean-css-4.2.3.tgzMIT
clipboard.jsclipboard-2.0.0.min.jsMIT
cni-plugincni-plugin-v3.16.6Tigera Proprietary
@babel/code-framecode-frame-7.10.1.tgzMIT
codemirrorcodemirror-5.57.0.jsMIT
codemirrorcodemirror-5.57.0.tgzMIT
collection-visitcollection-visit-1.0.0.tgzMIT
color-convertcolor-convert-1.9.3.tgzMIT
color-namecolor-name-1.1.3.tgzMIT
commandercommander-2.13.0.tgzMIT
commandercommander-2.20.3.tgzMIT
commandercommander-4.1.1.tgzMIT
component-emittercomponent-emitter-1.3.0.tgzMIT
concat-mapconcat-map-0.0.1.tgzMIT
configparserconfigparser-4.0.2-py2.py3-none-any.whlMIT
connected-react-routerconnected-react-router-6.5.2.tgzMIT
contextlib2contextlib2-0.6.0.post1-py2.py3-none-any.whlPython 2.0
copy-descriptorcopy-descriptor-0.1.1.tgzMIT
@popperjs/corecore-2.4.4.tgzMIT
core-jscore-js-1.2.7.tgzMIT
core-jscore-js-2.5.1.tgzMIT
core-jscore-js-2.5.7.tgzMIT
core-jscore-js-3.6.5.tgzMIT
cose-basecose-base-1.0.3.tgzMIT
create-react-classcreate-react-class-15.6.2.tgzMIT
css-box-modelcss-box-model-1.1.1.tgzMIT
css-selectcss-select-1.2.0.tgzBSD 2
css-whatcss-what-2.1.3.tgzBSD 2
@ungap/custom-elementscustom-elements-0.1.12.tgzISC
cytoscapecytoscape-3.15.2.tgzMIT
cytoscapecytoscape-3.18.0.min.jsMIT
cytoscape-avsdfcytoscape-avsdf-1.0.0.tgzMIT
cytoscape-cisecytoscape-cise-1.0.0.tgzMIT
cytoscape-context-menuscytoscape-context-menus-4.0.0.tgzMIT
cytoscape-cose-bilkentcytoscape-cose-bilkent-4.1.0.tgzMIT
cytoscape-dagre-cluster-fixcytoscape-dagre-cluster-fix-2.2.5.tgzMIT
cytoscape-expand-collapsecytoscape-expand-collapse-4.0.0.tgzMIT
cytoscape-fcosecytoscape-fcose-1.2.3.tgzMIT
cytoscape-layerscytoscape-layers-2.1.0.tgzMIT
cytoscape-navigatorcytoscape-navigator-2.0.1.tgzMIT
cytoscape-poppercytoscape-popper-1.0.7.jsMIT
cytoscape-poppercytoscape-popper-1.0.7.tgzMIT
d3d3-5.5.0.tgzBSD 3
d3-arrayd3-array-1.2.1.tgzBSD 3
d3-arrayd3-array-1.2.4.tgzBSD 3
d3-axisd3-axis-1.0.8.tgzBSD 3
d3-brushd3-brush-1.0.4.tgzBSD 3
d3-chordd3-chord-1.0.4.tgzBSD 3
d3-collectiond3-collection-1.0.4.tgzBSD 3
d3-collectiond3-collection-1.0.7.tgzBSD 3
d3-colord3-color-1.0.3.tgzBSD 3
d3-contourd3-contour-1.2.0.tgzBSD 3
d3-dispatchd3-dispatch-1.0.3.tgzBSD 3
d3-dragd3-drag-1.2.1.tgzBSD 3
d3-dsvd3-dsv-1.0.8.tgzBSD 3
d3-eased3-ease-1.0.3.tgzBSD 3
d3-fetchd3-fetch-1.1.0.tgzBSD 3
d3-forced3-force-1.1.0.tgzBSD 3
d3-formatd3-format-1.2.2.tgzBSD 3
d3-geod3-geo-1.10.0.tgzBSD 3
d3-hierarchyd3-hierarchy-1.1.6.tgzBSD 3
d3-interpolated3-interpolate-1.1.6.tgzBSD 3
d3-interpolated3-interpolate-1.3.2.tgzBSD 3
d3-pathd3-path-1.0.5.tgzBSD 3
d3-polygond3-polygon-1.0.3.tgzBSD 3
d3-quadtreed3-quadtree-1.0.3.tgzBSD 3
d3-randomd3-random-1.1.0.tgzBSD 3
d3-sankey-circulard3-sankey-circular-0.34.0.tgzMIT
d3-scaled3-scale-2.0.0.tgzBSD 3
d3-scaled3-scale-2.1.2.tgzBSD 3
d3-scale-chromaticd3-scale-chromatic-1.3.0.tgzBSD 3
d3-selectiond3-selection-1.3.0.tgzBSD 3
d3-shaped3-shape-1.2.0.tgzBSD 3
d3-shaped3-shape-1.2.3.tgzBSD 3
d3-shaped3-shape-1.3.7.tgzBSD 3
d3-timed3-time-1.0.8.tgzBSD 3
d3-time-formatd3-time-format-2.1.1.tgzBSD 3
d3-timerd3-timer-1.0.7.tgzBSD 3
d3-transitiond3-transition-1.1.1.tgzBSD 3
d3-voronoid3-voronoi-1.1.2.tgzBSD 3
d3-zoomd3-zoom-1.7.1.tgzBSD 3
dagredagre-0.7.4.jsMIT
dagre-cluster-fixdagre-cluster-fix-0.9.3.tgzMIT
debugdebug-2.6.9.tgzMIT
decimal.js-lightdecimal.js-light-2.5.0.tgzMIT
decode-uri-componentdecode-uri-component-0.2.0.tgzMIT
deepdiffdeepdiff-3.3.0-py2-none-any.whlMIT
deepmergedeepmerge-2.1.1.tgzMIT
define-propertiesdefine-properties-1.1.2.tgzMIT
define-propertiesdefine-properties-1.1.3.tgzMIT
define-propertydefine-property-0.2.5.tgzMIT
define-propertydefine-property-1.0.0.tgzMIT
define-propertydefine-property-2.0.2.tgzMIT
diffdiff-3.5.0.tgzBSD 3
diff2htmldiff2html-2.4.0.tgzMIT
dom-converterdom-converter-0.2.0.tgzMIT
dom-helpersdom-helpers-3.3.1.tgzMIT
dom-helpersdom-helpers-3.4.0.tgzMIT
dom-serializerdom-serializer-0.2.2.tgzMIT
dom-walkdom-walk-0.1.2.tgzMIT
domelementtypedomelementtype-1.3.1.tgzBSD 2
domelementtypedomelementtype-2.0.1.tgzBSD 2
domhandlerdomhandler-2.4.2.tgzBSD 2
domutilsdomutils-1.5.1.tgzBSD 2
domutilsdomutils-1.7.0.tgzBSD 2
dot-casedot-case-3.0.3.tgzMIT
elasticsearchelasticsearch-6.8.1-py2.py3-none-any.whlApache 2.0
elementary-circuits-directed-graphelementary-circuits-directed-graph-1.2.0.tgzMIT
emojis-listemojis-list-3.0.0.tgzMIT
encodingencoding-0.1.12.tgzMIT
entitiesentities-1.1.2.tgzBSD 2
entitiesentities-2.0.3.tgzBSD 2
es-abstractes-abstract-1.17.5.tgzMIT
es-to-primitivees-to-primitive-1.2.1.tgzMIT
es5-shimes5-shim-4.3.1.jsMIT
escape-string-regexpescape-string-regexp-1.0.5.tgzMIT
esprimaesprima-4.0.0.tgzBSD 2
expand-bracketsexpand-brackets-2.1.4.tgzMIT
extend-shallowextend-shallow-2.0.1.tgzMIT
extend-shallowextend-shallow-3.0.2.tgzMIT
extglobextglob-2.0.4.tgzMIT
fast-levenshteinfast-levenshtein-2.0.6.tgzMIT
fbjsfbjs-0.8.16.tgzMIT
felixfelix-v3.17.2Tigera Proprietary
file-saverfile-saver-2.0.1.tgzMIT
fill-rangefill-range-4.0.0.tgzMIT
@fortawesome/fontawesome-common-typesfontawesome-common-types-0.2.32.tgzMIT
@fortawesome/fontawesome-svg-corefontawesome-svg-core-1.2.10.tgzMIT
for-infor-in-1.0.2.tgzMIT
foreachforeach-2.0.5.tgzMIT
fork-ts-checker-webpack-pluginfork-ts-checker-webpack-plugin-4.1.6.tgzMIT
formikformik-2.1.3.tgzMIT
fragment-cachefragment-cache-0.2.1.tgzMIT
@fortawesome/free-brands-svg-iconsfree-brands-svg-icons-5.6.1.tgzCC BY 4.0
@fortawesome/free-regular-svg-iconsfree-regular-svg-icons-5.15.1.tgzCC BY 4.0
@fortawesome/free-solid-svg-iconsfree-solid-svg-icons-5.6.1.tgzCC BY 4.0
funcsigsfuncsigs-1.0.2-py2.py3-none-any.whlApache 2.0
function-bindfunction-bind-1.1.1.tgzMIT
get-valueget-value-2.0.6.tgzMIT
github.com/alecthomas/participlegithub.com/alecthomas/participle-v0.3.0MIT
github.com/apparentlymart/go-cidr/cidrgithub.com/apparentlymart/go-cidr/cidr-v1.0.1MIT
github.com/aquasecurity/kube-bench/checkgithub.com/aquasecurity/kube-bench/check-v0.0.34Apache 2.0
github.com/araddon/dateparsegithub.com/araddon/dateparse-262228af701ebf3932b8b8488da6781b9d585c88MIT
github.com/avast/retry-gogithub.com/avast/retry-go-v2.2.0MIT
github.com/aws/aws-lambda-go/eventsgithub.com/aws/aws-lambda-go/events-v1.13.3Apache 2.0
github.com/aws/aws-sdk-go/awsgithub.com/aws/aws-sdk-go/aws-v1.25.8Apache 2.0
github.com/bmizerany/patgithub.com/bmizerany/pat-6226ea591a40176dd3ff9cd8eff81ed6ca721a00MIT
github.com/bronze1man/gostrongswanvicigithub.com/bronze1man/gostrongswanvici-27d02f80ba4008de552efb746b3f6eaa7718b518MIT
github.com/buger/jsonparsergithub.com/buger/jsonparser-v1.0.0MIT
github.com/burntsushi/tomlgithub.com/burntsushi/toml-v0.3.1MIT
github.com/caimeo/iniflagsgithub.com/caimeo/iniflags-ef4ae6c5cd79d20db0b18bc5ebd8657fac7260e5BSD 2
github.com/cloudflare/cfssl/loggithub.com/cloudflare/cfssl/log-v1.4.1BSD 2
github.com/containernetworking/cni/libcnigithub.com/containernetworking/cni/libcni-v0.8.0Apache 2.0
github.com/containernetworking/plugins/pkg/hnsgithub.com/containernetworking/plugins/pkg/hns-v0.8.5Apache 2.0
github.com/coreos/go-oidcgithub.com/coreos/go-oidc-v2.1.0Apache 2.0
github.com/coreos/go-semver/semvergithub.com/coreos/go-semver/semver-v0.3.0Apache 2.0
github.com/davecgh/go-spew/spewgithub.com/davecgh/go-spew/spew-v1.1.1ISC
github.com/docker/docker/api/typesgithub.com/docker/docker/api/types-v1.13.1Apache 2.0
github.com/docker/docker/clientgithub.com/docker/docker/client-v1.13.1Apache 2.0
github.com/docker/go-connections/natgithub.com/docker/go-connections/nat-v0.4.0Apache 2.0
github.com/docopt/docopt-gogithub.com/docopt/docopt-go-ee0de3bc6815ee19d4a46c7eb90f829db0e014b1MIT
github.com/elastic/go-elasticsearch/v7github.com/elastic/go-elasticsearch/v7-v7.3.0Apache 2.0
github.com/envoyproxy/data-plane-api/envoy/api/v2/coregithub.com/envoyproxy/data-plane-api/envoy/api/v2/core-ffd420ef8a9ad148642236aa6d89e2855b41c821Apache 2.0
github.com/fsnotify/fsnotifygithub.com/fsnotify/fsnotify-v1.4.9BSD 3
github.com/gavv/monotimegithub.com/gavv/monotime-30dba43534243e3484a34676a0f068d12b989f84Apache 2.0
github.com/getlantern/deepcopygithub.com/getlantern/deepcopy-v1Apache 2.0
github.com/ghodss/yamlgithub.com/ghodss/yaml-v1.0.0BSD 3
github.com/go-ini/inigithub.com/go-ini/ini-v1.43.0Apache 2.0
github.com/go-logr/logrgithub.com/go-logr/logr-v0.3.0Apache 2.0
github.com/go-openapi/specgithub.com/go-openapi/spec-v0.19.3Apache 2.0
github.com/go-sql-driver/mysqlgithub.com/go-sql-driver/mysql-v1.4.1Mozilla 2.0
github.com/gofrs/flockgithub.com/gofrs/flock-v0.8.0BSD 3
github.com/gogo/googleapis/google/rpcgithub.com/gogo/googleapis/google/rpc-v1.2.0Apache 2.0
github.com/gogo/protobuf/protogithub.com/gogo/protobuf/proto-v1.3.1BSD 3
github.com/golang-collections/collections/stackgithub.com/golang-collections/collections/stack-604e922904d35e97f98a774db7881f049cd8d970MIT
github.com/google/go-cmp/cmpgithub.com/google/go-cmp/cmp-v0.4.0BSD 3
github.com/google/gofuzzgithub.com/google/gofuzz-v1.1.0Apache 2.0
github.com/google/gopacketgithub.com/google/gopacket-v1.1.18BSD 3
github.com/google/netstack/tcpip/headergithub.com/google/netstack/tcpip/header-55fcc16cd0eb096d8418f7bc5162483c31a4e82bApache 2.0
github.com/hashicorp/go-versiongithub.com/hashicorp/go-version-v1.2.1Mozilla 2.0
github.com/hashicorp/golang-lrugithub.com/hashicorp/golang-lru-v0.5.1Mozilla 2.0
github.com/hashicorp/yamuxgithub.com/hashicorp/yamux-2f1d1f20f75d5404f53b9edf6b53ed5505508675Mozilla 2.0
github.com/howeyc/fsnotifygithub.com/howeyc/fsnotify-v0.9.0BSD 3
github.com/hpcloud/tailgithub.com/hpcloud/tail-v1.0.0MIT
github.com/ishidawataru/sctpgithub.com/ishidawataru/sctp-00ab2ac2db07a138417639ef3f39672c65dbb9a0BSD 3
github.com/jarcoal/httpmockgithub.com/jarcoal/httpmock-v1.0.5MIT
github.com/jinzhu/copiergithub.com/jinzhu/copier-v0.1.0MIT
github.com/jmespath/go-jmespathgithub.com/jmespath/go-jmespath-0.3.0Apache 2.0
github.com/joho/godotenvgithub.com/joho/godotenv-v1.3.0MIT
github.com/jpillora/backoffgithub.com/jpillora/backoff-v1.0.0MIT
github.com/json-iterator/gogithub.com/json-iterator/go-v1.1.10MIT
github.com/juju/clockgithub.com/juju/clock-9c5c9712527c7986f012361e7d13756b4d99543dLGPL 3.0
github.com/juju/errorsgithub.com/juju/errors-3fe23663418fc1d724868c84f21b7519bbac7441LGPL 3.0
github.com/juju/mutexgithub.com/juju/mutex-d21b13acf4bfd8a8b0482a3a78e44d98880b40d3LGPL 3.0
github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utilsgithub.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/utils-v1.1.0Apache 2.0
github.com/kardianos/osextgithub.com/kardianos/osext-2bc1f35cddc0cc527b4bc3dce8578fc2a6c11384BSD 3
github.com/kelseyhightower/envconfiggithub.com/kelseyhightower/envconfig-v1.4.0MIT
github.com/kelseyhightower/memkvgithub.com/kelseyhightower/memkv-v0.1.1MIT
github.com/konsorten/go-windows-terminal-sequencesgithub.com/konsorten/go-windows-terminal-sequences-v1.0.1MIT
github.com/lestrrat-go/file-rotatelogsgithub.com/lestrrat-go/file-rotatelogs-v2.4.0MIT
github.com/libp2p/go-reuseportgithub.com/libp2p/go-reuseport-v0.0.1ISC
github.com/lithammer/dedentgithub.com/lithammer/dedent-v1.1.0MIT
github.com/mailru/easyjsongithub.com/mailru/easyjson-v0.7.0MIT
github.com/masterminds/spriggithub.com/masterminds/sprig-v2.19.0MIT
github.com/mcuadros/go-versiongithub.com/mcuadros/go-version-92cdf37c5b7579ebaf7a036da94b40995972088dMIT
github.com/microsoft/hcsshimgithub.com/microsoft/hcsshim-v0.8.6MIT
github.com/mipearson/rfwgithub.com/mipearson/rfw-6f0a6f3266ba1058df9ef0c94cda1cecd2e62852MIT
github.com/mitchellh/go-homedirgithub.com/mitchellh/go-homedir-v1.1.0MIT
github.com/modern-go/concurrentgithub.com/modern-go/concurrent-1.0.3Apache 2.0
github.com/modern-go/reflect2github.com/modern-go/reflect2-v1.0.1Apache 2.0
github.com/natefinch/atomicgithub.com/natefinch/atomic-a62ce929ffcc871a51e98c6eba7b20321e3ed62dMIT
github.com/nmrshll/go-cpgithub.com/nmrshll/go-cp-61436d3b7cfa1bc1e8e455c35d8f60b8e51ccc2eMIT
github.com/nxadm/tailgithub.com/nxadm/tail-v1.4.4MIT
github.com/olekukonko/tablewritergithub.com/olekukonko/tablewriter-v0.0.2MIT
github.com/olivere/elastic/v7github.com/olivere/elastic/v7-v7.0.6MIT
github.com/onsi/ginkgogithub.com/onsi/ginkgo-v1.15.0MIT
github.com/onsi/gomegagithub.com/onsi/gomega-v1.7.0MIT
github.com/openshift/api/config/v1github.com/openshift/api/config/v1-d0822898eabb929c40c5146116252477abab8d18Apache 2.0
github.com/openshift/library-go/pkg/cryptogithub.com/openshift/library-go/pkg/crypto-9350cd67a9110bcaf9a85d391fa264afbbff1342Apache 2.0
github.com/osrg/gobgp/clientgithub.com/osrg/gobgp/client-v1.22Apache 2.0
github.com/paloaltonetworks/pangogithub.com/paloaltonetworks/pango-v0.1.1ISC
github.com/patrickmn/go-cachegithub.com/patrickmn/go-cache-v2.1.0MIT
github.com/pkg/errorsgithub.com/pkg/errors-v0.8.1BSD 2
github.com/projectcalico/cni-plugingithub.com/projectcalico/cni-pluginApache 2.0
github.com/projectcalico/felixgithub.com/projectcalico/felixApache 2.0
github.com/projectcalico/go-jsongithub.com/projectcalico/go-json/json-6219dc7339ba20ee4c57df0a8baac62317d19cb1BSD 2
github.com/projectcalico/go-yaml-wrappergithub.com/projectcalico/go-yaml-wrapper-090425220c545f6d179db17af395f5aac30b6926BSD 3
kube-controllerskube-controllersTigera Proprietary
github.com/projectcalico/libcalico-gogithub.com/projectcalico/libcalico-go/libApache 2.0
github.com/projectcalico/pod2daemongithub.com/projectcalico/pod2daemonApache 2.0
typhatyphaTigera Proprietary
github.com/prometheus/client_golang/github.com/prometheus/client_golang/prometheus-v1.7.1Apache 2.0
github.com/rakelkar/gonetsh/netshgithub.com/rakelkar/gonetsh/netsh-e5c5ffe4bdf04bc060fc45ff4aca2349f51c94a7Apache 2.0
github.com/robfig/crongithub.com/robfig/cron-v1.2.0MIT
github.com/satori/go.uuidgithub.com/satori/go.uuid-v1.2.0MIT
github.com/shirou/gopsutil/processgithub.com/shirou/gopsutil/process-v2.19.03BSD 3
github.com/sirupsen/logrusgithub.com/sirupsen/logrus-v1.4.2MIT
github.com/sirupsen/logrusgithub.com/sirupsen/logrus-v1.6.0MIT
github.com/spf13/cobragithub.com/spf13/cobra-v0.0.3Apache 2.0
github.com/spf13/cobragithub.com/spf13/cobra-v1.0.0Apache 2.0
github.com/spf13/pflaggithub.com/spf13/pflag-v1.0.5BSD 3
github.com/spf13/vipergithub.com/spf13/viper-v1.6.1MIT
github.com/stretchr/testifygithub.com/stretchr/testify/mock-v1.4.0MIT
github.com/termie/go-shutilgithub.com/termie/go-shutil-bcacb06fecaeec8dc42af03c87c6949f4a05c74cMIT
github.com/tigera/apigithub.com/tigera/api/Apache 2.0
github.com/vishvananda/netlinkgithub.com/vishvananda/netlink-v1.1.0Apache 2.0
github.com/willf/bitsetgithub.com/willf/bitset-v1.1.11BSD 3
github.com/workiva/go-datastructures/trie/ctriegithub.com/workiva/go-datastructures/trie/ctrie-v1.0.50Apache 2.0
github.com/x-cray/logrus-prefixed-formattergithub.com/x-cray/logrus-prefixed-formatter-v0.5.2MIT
github.com/yalp/jsonpathgithub.com/yalp/jsonpath-5cc68e5049a040829faef3a44c00ec4332f6dec7BSD 3
globalglobal-4.4.0.tgzMIT
go.etcd.io/etcd/go.etcd.io/etcd/client-v0.5.0-alpha.5.0.20201125193152-8a03d2e9614bApache 2.0
go.uber.org/zapgo.uber.org/zap-v1.15.0MIT
golang.org/x/cryptogolang.org/x/crypto/BSD 3
golang.org/x/netgolang.org/x/net/BSD 3
golang.org/x/sync/golang.org/x/sync/BSD 3
golang.org/x/sysgolang.org/x/sysBSD 3
github.com/golang/textgolang.org/x/textGolang BSD + Patents
golang.zx2c4.com/wireguard/golang.zx2c4.com/wireguard/MIT
google-libphonenumbergoogle-libphonenumber-3.2.2.tgzApache 2.0
google.golang.org/grpcgoogle.golang.org/grpc-v1.27.0Apache 2.0
google-authgoogle_auth-1.26.1-py2.py3-none-any.whlApache 2.0
gopkg.in/fsnotify/fsnotify.v1gopkg.in/fsnotify/fsnotify.v1-v1.4.7BSD 3
gopkg.in/go-playground/validator.v9gopkg.in/go-playground/validator.v9-v9.30.2MIT
gopkg.in/inf.v0gopkg.in/inf.v0-v0.9.0BSD 3
gopkg.in/natefinch/lumberjack.v2gopkg.in/natefinch/lumberjack.v2-v2.0.0MIT
gopkg.in/square/go-jose.v2gopkg.in/square/go-jose.v2-v2.2.3Apache 2.0
gopkg.in/tchap/go-patricia.v2/patriciagopkg.in/tchap/go-patricia.v2/patricia-v2.3.0MIT
gopkg.in/tomb.v1gopkg.in/tomb.v1-dd632973f1e7218eb1089048e0798ec9ae7dceb8BSD 3
gopkg.in/yaml.v2gopkg.in/yaml.v2-v2.4.0Apache 2.0
graphlibgraphlib-2.1.8.tgzMIT
gudgud-1.0.0.tgzMIT
hashas-1.0.3.tgzMIT
has-flaghas-flag-3.0.0.tgzMIT
has-symbolshas-symbols-1.0.1.tgzMIT
has-valuehas-value-0.3.1.tgzMIT
has-valuehas-value-1.0.0.tgzMIT
has-valueshas-values-0.1.4.tgzMIT
has-valueshas-values-1.0.0.tgzMIT
hehe-1.2.0.tgzMIT
heapheap-0.2.6.tgzPython 2.0
@babel/helper-validator-identifierhelper-validator-identifier-7.10.1.tgzMIT
@babel/highlighthighlight-7.10.1.tgzMIT
historyhistory-4.9.0.tgzMIT
twitterhogan.jsApache 2.0
hogan.jshogan.js-3.0.2.tgzApache 2.0
hoist-non-react-staticshoist-non-react-statics-3.1.0.tgzBSD 3
hoist-non-react-staticshoist-non-react-statics-3.3.0.tgzBSD 3
@types/html-minifier-terserhtml-minifier-terser-5.1.0.tgzMIT
html-minifier-terserhtml-minifier-terser-5.1.1.tgzMIT
html-webpack-pluginhtml-webpack-plugin-4.3.0.tgzMIT
htmlparser2htmlparser2-3.10.1.tgzMIT
humpshumps-2.0.1.tgzMIT
icepickicepick-1.3.0.tgzMIT
iconv-liteiconv-lite-0.4.23.tgzMIT
idnaidna-2.10-py2.py3-none-any.whlBSD 3
idnaidna-2.7-py2.py3-none-any.whlBSD 2
immutableimmutable-3.8.2.tgzMIT
importlib-metadataimportlib_metadata-2.1.1-py2.py3-none-any.whlApache 2.0
inheritsinherits-2.0.4.tgzISC
instantsearch.jsinstantsearch.production-4.4.1.min.jsMIT
invariantinvariant-2.2.2.tgzBSD 3
invariantinvariant-2.2.4.tgzMIT
ip-regexip-regex-2.1.0.tgzMIT
ip-regexip-regex-4.1.0.tgzMIT
ipaddr.jsipaddr.js-1.9.1.tgzMIT
ipaddressipaddress-1.0.23-py2.py3-none-any.whlPython 2.0
is-accessor-descriptoris-accessor-descriptor-0.1.6.tgzMIT
is-accessor-descriptoris-accessor-descriptor-1.0.0.tgzMIT
is-bufferis-buffer-1.1.6.tgzMIT
is-callableis-callable-1.1.5.tgzMIT
is-cidris-cidr-3.1.0.tgzBSD 2
is-data-descriptoris-data-descriptor-0.1.4.tgzMIT
is-data-descriptoris-data-descriptor-1.0.0.tgzMIT
is-date-objectis-date-object-1.0.1.tgzMIT
is-descriptoris-descriptor-0.1.6.tgzMIT
is-descriptoris-descriptor-1.0.2.tgzMIT
is-extendableis-extendable-0.1.1.tgzMIT
is-extendableis-extendable-1.0.1.tgzMIT
is-ipis-ip-3.1.0.tgzMIT
is-numberis-number-3.0.0.tgzMIT
is-plain-objectis-plain-object-2.0.4.tgzMIT
is-regexis-regex-1.0.5.tgzMIT
is-streamis-stream-1.1.0.tgzMIT
is-symbolis-symbol-1.0.3.tgzMIT
is-windowsis-windows-1.0.2.tgzMIT
isarrayisarray-0.0.1.tgzMIT
isarrayisarray-1.0.0.tgzMIT
isobjectisobject-2.1.0.tgzMIT
isobjectisobject-3.0.1.tgzMIT
isomorphic-fetchisomorphic-fetch-2.2.1.tgzMIT
jqueryjquery-2.2.0.min.jsMIT
jqueryjquery-3.4.0.min.jsMIT
js-cookiejs-cookie-2.2.1.tgzMIT
js-tokensjs-tokens-3.0.2.tgzMIT
js-tokensjs-tokens-4.0.0.tgzMIT
js-yamljs-yaml-3.14.0.tgzMIT
jsanjsan-3.1.13.tgzMIT
json5json5-1.0.1.tgzMIT
jsonpicklejsonpickle-2.0.0-py2.py3-none-any.whlBSD 2
jsrsasignjsrsasign-5.1.0.tgzMIT
k8s.io/apik8s.io/apiApache 2.0
k8s.io/apiextensions-apiserver/k8s.io/apiextensions-apiserver/Apache 2.0
k8s.io/apimachineryk8s.io/apimachinery/Apache 2.0
k8s.io/apiserverk8s.io/apiserverApache 2.0
k8s.io/client-gok8s.io/client-go/Apache 2.0
k8s.io/component-base/k8s.io/component-base/Apache 2.0
k8s.io/klogk8s.io/klog-v1.0.0Apache 2.0
k8s.io/kube-aggregator/k8s.io/kube-aggregator/Apache 2.0
k8s.io/kube-openapi/k8s.io/kube-openapiApache 2.0
k8s.io/kubernetes/k8s.io/kubernetes/Apache 2.0
k8s.io/utils/stringsk8s.io/utils/stringsApache 2.0
keycodekeycode-2.1.9.tgzMIT
kind-ofkind-of-3.2.2.tgzMIT
kind-ofkind-of-4.0.0.tgzMIT
kind-ofkind-of-5.1.0.tgzMIT
kind-ofkind-of-6.0.3.tgzMIT
kube-controllerskube-controllers-v3.0.11Tigera Proprietary
kube-controllerskube-controllers-v3.16.6Tigera Proprietary
kuberneteskubernetes-12.0.1-py2.py3-none-any.whlApache 2.0
layout-baselayout-base-1.0.2.tgzMIT
projectcalicolibcalico-go-v3.18.0-0.devApache 2.0
loader-utilsloader-utils-1.4.0.tgzMIT
lodashlodash-4.17.19.tgzMIT
lodashlodash-4.17.20.tgzMIT
lodash-eslodash-es-4.17.15.tgzMIT
lodash.debouncelodash.debounce-4.0.8.tgzMIT
lodash.getlodash.get-4.4.2.tgzMIT
lodash.isequallodash.isequal-4.5.0.tgzMIT
lodash.throttlelodash.throttle-4.1.1.tgzMIT
lodash.topathlodash.topath-4.5.2.tgzMIT
pimterryloglevel-v1.6.8MIT
loose-envifyloose-envify-1.3.1.tgzMIT
loose-envifyloose-envify-1.4.0.tgzMIT
lower-caselower-case-2.0.1.tgzMIT
map-cachemap-cache-0.2.2.tgzMIT
map-visitmap-visit-1.0.0.tgzMIT
math-expression-evaluatormath-expression-evaluator-1.2.17.tgzMIT
megacubomegacubo-br-Megacubo_15.4.7_linux_ia32LGPL 2.1
memoize-onememoize-one-4.0.3.tgzMIT
microevent.tsmicroevent.ts-0.1.1.tgzMIT
micromatchmicromatch-3.1.10.tgzMIT
min-documentmin-document-2.19.0.tgzMIT
mini-create-react-contextmini-create-react-context-0.3.2.tgzMIT
minimatchminimatch-3.0.4.tgzISC
minimistminimist-1.2.5.tgzMIT
mixin-deepmixin-deep-1.3.2.tgzMIT
mkdirpmkdirp-0.3.0.tgzMIT X11
mochamocha-1.6.0.jsMIT
momentmoment-2.22.2.tgzMIT
more-itertoolsmore_itertools-5.0.0-py2-none-any.whlMIT
msms-2.0.0.tgzMIT
nanoidnanoid-2.1.7.tgzMIT
nanomatchnanomatch-1.2.13.tgzMIT
netaddrnetaddr-0.7.19-py2.py3-none-any.whlBSD 3
no-caseno-case-3.0.3.tgzMIT
@types/nodenode-9.3.0.tgzMIT
node-fetchnode-fetch-1.7.3.tgzMIT
nodejsnode-v10.23.1Node.js
nodenode-v3.17.2Tigera Proprietary
noptnopt-1.0.10.tgzMIT
nose-timernose-timer-0.7.1.tar.gzMIT
nose-parameterizednose_parameterized-0.6.0-py2.py3-none-any.whlBSD 3
nth-checknth-check-1.0.2.tgzBSD 2
oauthliboauthlib-3.1.0-py2.py3-none-any.whlBSD 3
object-assignobject-assign-4.1.1.tgzMIT
object-copyobject-copy-0.1.0.tgzMIT
object-inspectobject-inspect-1.7.0.tgzMIT
object-keysobject-keys-1.0.11.tgzMIT
object-keysobject-keys-1.1.1.tgzMIT
object-visitobject-visit-1.0.1.tgzMIT
object.assignobject.assign-4.1.0.tgzMIT
object.getownpropertydescriptorsobject.getownpropertydescriptors-2.1.0.tgzMIT
object.pickobject.pick-1.3.0.tgzMIT
oidc-clientoidc-client-1.4.1.tgzApache 2.0
openshiftorigin-v3.6.1Apache 2.0
packagingpackaging-20.9-py2.py3-none-any.whlBSD 2
param-caseparam-case-3.0.3.tgzMIT
parse-durationparse-duration-0.4.4.tgzMIT
pascal-casepascal-case-3.1.1.tgzMIT
pascalcasepascalcase-0.1.1.tgzMIT
path-to-regexppath-to-regexp-1.7.0.tgzMIT
pathlib2pathlib2-2.3.5-py2.py3-none-any.whlMIT
pegjspegjs-0.10.0.tgzMIT
performance-nowperformance-now-2.1.0.tgzMIT
pluggypluggy-0.13.1-py2.py3-none-any.whlMIT
popper.jspopper-1.16.0.jsMIT
popper.jspopper.js-1.16.1.tgzMIT
posix-character-classesposix-character-classes-0.1.1.tgzMIT
pretty-errorpretty-error-2.1.1.tgzMIT
processprocess-0.11.10.tgzMIT
promisepromise-7.3.1.tgzMIT
prop-typesprop-types-15.5.10.jsBSD 3
prop-typesprop-types-15.6.0.tgzMIT
prop-typesprop-types-15.6.2.jsBSD 3
prop-typesprop-types-15.6.2.tgzMIT
prop-typesprop-types-15.7.2.tgzMIT
prop-types-extraprop-types-extra-1.0.1.tgzMIT
pypy-1.10.0-py2.py3-none-any.whlMIT
pyasn1pyasn1-0.4.8-py2.py3-none-any.whlBSD 2
pyasn1-modulespyasn1_modules-0.2.8-py2.py3-none-any.whlBSD 2
pyparsingpyparsing-2.4.7-py2.py3-none-any.whlMIT
pytestpytest-4.6.11-py2.py3-none-any.whlMIT
python-dateutilpython_dateutil-2.8.1-py2.py3-none-any.whlBSD 3
PyYAMLPyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whlMIT
rafraf-3.4.0.tgzMIT
raf-schdraf-schd-4.0.0.tgzMIT
raven-jsraven-js-3.22.1.tgzBSD 2
reactreact-15.6.1.jsMIT
reactreact-16.13.1.tgzMIT
reactreact-16.8.6.tgzMIT
react-beautiful-dndreact-beautiful-dnd-10.0.2.tgzApache 2.0
react-bootstrapreact-bootstrap-0.32.1.tgzMIT
react-codemirror2react-codemirror2-5.1.0.tgzMIT
react-confirmreact-confirm-0.1.16.tgzMIT
react-cytoscapejsreact-cytoscapejs-1.2.1.tgzMIT
plotlyreact-cytoscapejs-v1.2.1MIT
react-day-pickerreact-day-picker-7.4.8.tgzMIT
react-domreact-dom-15.6.1.jsMIT
react-domreact-dom-16.13.1.tgzMIT
react-domreact-dom-16.8.6.tgzMIT
react-draggablereact-draggable-3.0.5.tgzMIT
react-fast-comparereact-fast-compare-2.0.4.tgzMIT
react-filter-boxreact-filter-box-3.4.1.tgzMIT
@fortawesome/react-fontawesomereact-fontawesome-0.1.3.tgzMIT
react-grid-layoutreact-grid-layout-0.16.3.tgzMIT
react-hot-loaderreact-hot-loader-4.12.21.tgzMIT
react-input-autosizereact-input-autosize-2.1.2.tgzMIT
JedWatsonreact-input-autosize-v2.1.2MIT
react-isreact-is-16.6.3.tgzMIT
react-isreact-is-16.8.6.tgzMIT
react-isreact-is-16.9.0.tgzMIT
react-json-prettyreact-json-pretty-2.2.0.tgzMIT
react-lifecycles-compatreact-lifecycles-compat-3.0.4.tgzMIT
react-native-segmented-control-tabreact-native-segmented-control-tab-3.2.1.tgzMIT
react-new-windowreact-new-window-0.1.2.tgzMIT
react-notification-systemreact-notification-system-0.2.17.tgzMIT
react-overlaysreact-overlays-0.8.3.tgzMIT
react-prop-typesreact-prop-types-0.4.0.tgzMIT
react-querybuilderreact-querybuilder-3.0.0.tgzMIT
react-reduxreact-redux-5.1.1.tgzMIT
react-reduxreact-redux-7.1.1.tgzMIT
react-redux-formreact-redux-form-1.16.5.tgzMIT
react-resizablereact-resizable-1.7.5.tgzMIT
react-resize-detectorreact-resize-detector-2.3.0.tgzMIT
react-routerreact-router-5.0.1.tgzMIT
react-router-domreact-router-dom-5.0.1.tgzMIT
react-showreact-show-2.0.4.tgzMIT
react-smoothreact-smooth-1.0.2.tgzMIT
react-split-panereact-split-pane-0.1.92.tgzMIT
react-style-proptypereact-style-proptype-3.2.2.tgzMIT
react-switchreact-switch-5.0.0.tgzMIT
react-table-6react-table-6-6.11.0.tgzMIT
react-tablereact-table-7.5.1.tgzMIT
react-tooltipreact-tooltip-4.2.11.tgzMIT
react-transition-groupreact-transition-group-2.2.1.tgzBSD 3
react-transition-groupreact-transition-group-2.7.1.tgzBSD 3
readable-streamreadable-stream-3.6.0.tgzMIT
rechartsrecharts-1.5.0.tgzMIT
recharts-scalerecharts-scale-0.4.2.tgzMIT
reduce-css-calcreduce-css-calc-1.3.0.tgzMIT
reduce-function-callreduce-function-call-1.0.2.tgzMIT
reduxredux-4.0.1.tgzMIT
reduxredux-4.0.4.tgzMIT
redux-immutableredux-immutable-4.0.0.tgzBSD 3
redux-merge-immutable-reducersredux-merge-immutable-reducers-0.1.4.tgzMIT
redux-thunkredux-thunk-2.3.0.tgzMIT
reflect-metadatareflect-metadata-0.1.13.tgzApache 2.0
regenerator-runtimeregenerator-runtime-0.11.1.tgzMIT
regenerator-runtimeregenerator-runtime-0.12.1.tgzMIT
regenerator-runtimeregenerator-runtime-0.13.3.tgzMIT
regenerator-runtimeregenerator-runtime-0.13.5.tgzMIT
regex-notregex-not-1.0.2.tgzMIT
regexp-to-astregexp-to-ast-0.4.0.tgzMIT
relateurlrelateurl-0.2.7.tgzMIT
remotedev-serializeremotedev-serialize-0.1.8.tgzMIT
renderkidrenderkid-2.0.3.tgzMIT
repeat-elementrepeat-element-1.1.3.tgzMIT
repeat-stringrepeat-string-1.6.1.tgzMIT
requestsrequests-2.20.1-py2.py3-none-any.whlApache 2.0
requestsrequests-2.25.1-py2.py3-none-any.whlApache 2.0
requests-oauthlibrequests_oauthlib-1.3.0-py2.py3-none-any.whlISC
reselectreselect-2.5.4.tgzMIT
@juggle/resize-observerresize-observer-3.2.0.tgzApache 2.0
resize-observer-polyfillresize-observer-polyfill-1.5.1.tgzMIT
resolve-pathnameresolve-pathname-2.2.0.tgzMIT
resolve-urlresolve-url-0.2.1.tgzMIT
retret-0.1.15.tgzMIT
rsarsa-4.5-py2.py3-none-any.whlApache 2.0
@babel/runtimeruntime-7.1.5.tgzMIT
@babel/runtimeruntime-7.5.5.tgzMIT
@babel/runtime-corejs2runtime-corejs2-7.1.5.tgzMIT
rwrw-1.3.3.tgzBSD 3
safe-buffersafe-buffer-5.2.1.tgzMIT
safe-regexsafe-regex-1.1.0.tgzMIT
safer-buffersafer-buffer-2.1.2.tgzMIT
scandirscandir-1.10.0.tar.gzBSD 3
schedulerscheduler-0.13.6.tgzMIT
schedulerscheduler-0.18.0.tgzMIT
schedulerscheduler-0.19.1.tgzMIT
seamless-immutableseamless-immutable-7.1.4.tgzBSD 3
semversemver-5.7.1.tgzISC
set-valueset-value-2.0.1.tgzMIT
setimmediatesetimmediate-1.0.5.tgzMIT
setuptoolssetuptools-44.1.1-py2.py3-none-any.whlMIT
shallow-compareshallow-compare-1.2.2.tgzMIT
shallowequalshallowequal-1.1.0.tgzMIT
sigs.k8s.io/controller-runtimesigs.k8s.io/controller-runtime-v0.7.0Apache 2.0
sigs.k8s.io/kind/pkg/errorssigs.k8s.io/kind/pkg/errors-v0.9.0Apache 2.0
sigs.k8s.io/yamlsigs.k8s.io/yaml-v1.2.0BSD 3
simplejsonsimplejson-3.13.2.tar.gzAcademic 2.1
sixsix-1.15.0-py2.py3-none-any.whlMIT
snapdragonsnapdragon-0.8.2.tgzMIT
snapdragon-nodesnapdragon-node-2.1.1.tgzMIT
snapdragon-utilsnapdragon-util-3.0.1.tgzMIT
soupsievesoupsieve-1.9.6-py2.py3-none-any.whlMIT
@types/source-list-mapsource-list-map-0.1.2.tgzMIT
source-mapsource-map-0.5.7.tgzBSD 3
source-mapsource-map-0.6.1.tgzBSD 3
source-mapsource-map-0.7.3.tgzBSD 3
source-map-resolvesource-map-resolve-0.5.3.tgzMIT
source-map-supportsource-map-support-0.5.19.tgzMIT
source-map-urlsource-map-url-0.4.0.tgzMIT
split-stringsplit-string-3.1.0.tgzMIT
sprintfsprintf-1.0.3.jsBSD
sprintf-jssprintf-js-1.0.3.tgzBSD 3
static-extendstatic-extend-0.1.2.tgzMIT
string.prototype.trimendstring.prototype.trimend-1.0.1.tgzMIT
string.prototype.trimleftstring.prototype.trimleft-2.1.2.tgzMIT
string.prototype.trimrightstring.prototype.trimright-2.1.2.tgzMIT
string.prototype.trimstartstring.prototype.trimstart-1.0.1.tgzMIT
string_decoderstring_decoder-1.3.0.tgzMIT
strip-ansistrip-ansi-3.0.1.tgzMIT
strongly-connected-componentsstrongly-connected-components-1.0.1.tgzMIT
supports-colorsupports-color-5.5.0.tgzMIT
swagger-uiswagger-ui-bundle-3.37.0.jsApache 2.0
swagger-uiswagger-ui-standalone-preset-3.37.0.jsApache 2.0
symbol-observablesymbol-observable-1.2.0.tgzMIT
@types/tapabletapable-1.0.5.tgzMIT
tapabletapable-1.1.3.tgzMIT
JstarfishTechnical-Learning-609d9d75ca68e30aee8757b26f52bf132c644be7ISC
termcolortermcolor-1.1.0.tar.gzMIT
terserterser-4.7.0.tgzBSD 2
tiny-invarianttiny-invariant-1.0.3.tgzMIT
tiny-invarianttiny-invariant-1.0.6.tgzMIT
tiny-warningtiny-warning-1.0.2.tgzMIT
tiny-warningtiny-warning-1.0.3.tgzMIT
tippy.jstippy-bundle.iife-5.2.1.min.jsMIT
tippy.jstippy.js-6.2.5.tgzMIT
to-object-pathto-object-path-0.3.0.tgzMIT
to-regexto-regex-3.0.2.tgzMIT
to-regex-rangeto-regex-range-2.1.1.tgzMIT
tslibtslib-1.10.0.jsApache 2.0
tslibtslib-1.10.0.tgzApache 2.0
tslibtslib-1.13.0.jsApache 2.0
tslibtslib.es6-1.10.0.jsApache 2.0
tslibtslib.es6-1.13.0.jsApache 2.0
typescript-fsatypescript-fsa-2.5.0.tgzMIT
typescript-fsa-reducerstypescript-fsa-reducers-0.4.5.tgzMIT
ua-parser-jsua-parser-js-0.7.18.tgzMIT
@types/uglify-jsuglify-js-3.9.2.tgzMIT
uncontrollableuncontrollable-4.1.0.tgzMIT
union-valueunion-value-1.0.1.tgzMIT
unset-valueunset-value-1.0.0.tgzMIT
urixurix-0.1.0.tgzMIT
urllib3urllib3-1.24.3-py2.py3-none-any.whlMIT
urllib3urllib3-1.26.3-py2.py3-none-any.whlMIT
useuse-3.1.1.tgzMIT
util-deprecateutil-deprecate-1.0.2.tgzMIT
util.promisifyutil.promisify-1.0.0.tgzMIT
utilautila-0.4.0.tgzMIT
uuiduuid-7.0.3.tgzMIT
validatorvalidator-10.4.0.tgzMIT
value-equalvalue-equal-0.4.0.tgzMIT
warningwarning-3.0.0.tgzBSD 3
wcwidthwcwidth-0.2.5-py2.py3-none-any.whlMIT
@types/webpackwebpack-4.41.17.tgzMIT
@types/webpack-sourceswebpack-sources-1.4.0.tgzMIT
websocket_clientwebsocket_client-0.57.0-py2.py3-none-any.whlBSD 3
whatwg-fetchwhatwg-fetch-2.0.4.tgzMIT
worker-rpcworker-rpc-0.1.1.tgzMIT
zippzipp-1.2.0-py2.py3-none-any.whlMIT
github.com/projectcalico/birdgithub.com/projectcalico/bird/blob/v0.3.3GPL
egress-gatewayegress-gatewayTigera Proprietary
diff --git a/calico-cloud_versioned_docs/version-24-1/reference/component-resources/configuration.mdx b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/configuration.mdx new file mode 100644 index 0000000000..c515021fa0 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/configuration.mdx @@ -0,0 +1,574 @@ +--- +description: Reference for configuring the CNI plugin in Calico Cloud connected clusters covering operator-managed and manifest-mode CNI options. +--- + +# Configuring the Calico Cloud CNI plugins + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + + + +The $[prodname] CNI plugins do not need to be configured directly when installed by the operator. For a complete operator +configuration reference, see [the installation API reference documentation][installation]. + + + + +The $[prodname] CNI plugin is configured through the standard CNI +[configuration mechanism](https://github.com/containernetworking/cni/blob/master/SPEC.md#network-configuration) + +A minimal configuration file that uses $[prodname] for networking +and IPAM looks like this + +```json +{ + "name": "any_name", + "cniVersion": "0.1.0", + "type": "calico", + "ipam": { + "type": "calico-ipam" + } +} +``` + +If the `$[nodecontainer]` container on a node registered with a `NODENAME` other than the node hostname, the CNI plugin on this node must be configured with the same `nodename`: + +```json +{ + "name": "any_name", + "nodename": "", + "type": "calico", + "ipam": { + "type": "calico-ipam" + } +} +``` + +Additional configuration can be added as detailed below. + +## Generic + +### Datastore type + +The $[prodname] CNI plugin supports the following datastore: + +- `datastore_type` (kubernetes) + +### Logging + +Logging is always to `stderr`. Logs are also written to `/var/log/calico/cni/cni.log` on each host by default. + +Logging can be configured using the following options in the netconf. + +| Option name | Default | Description | +| -------------------- | ----------------------------- | --------------------------------------------------------------------------------------------------------- | +| `log_level` | INFO | Logging level. Allowed levels are `ERROR`, `WARNING`, `INFO`, and `DEBUG`. | +| `log_file_path` | `/var/log/calico/cni/cni.log` | Location on each host to write CNI log files to. Logging to file can be disabled by removing this option. | +| `log_file_max_size` | 100 | Max file size in MB log files can reach before they are rotated. | +| `log_file_max_age` | 30 | Max age in days that old log files will be kept on the host before they are removed. | +| `log_file_max_count` | 10 | Max number of rotated log files allowed on the host before they are cleaned up. | + +```json +{ + "name": "any_name", + "cniVersion": "0.1.0", + "type": "calico", + "log_level": "DEBUG", + "log_file_path": "/var/log/calico/cni/cni.log", + "ipam": { + "type": "calico-ipam" + } +} +``` + +### IPAM + +When using $[prodname] IPAM, the following flags determine what IP addresses should be assigned. NOTE: These flags are strings and not boolean values. + +- `assign_ipv4` (default: `"true"`) +- `assign_ipv6` (default: `"false"`) + +A specific IP address can be chosen by using [`CNI_ARGS`](https://github.com/appc/cni/blob/master/SPEC.md#parameters) and setting `IP` to the desired value. + +By default, $[prodname] IPAM will assign IP addresses from all the available IP pools. + +Optionally, the list of possible IPv4 and IPv6 pools can also be specified via the following properties: + +- `ipv4_pools`: An array of CIDR strings or pool names. (e.g., `"ipv4_pools": ["10.0.0.0/24", "20.0.0.0/16", "default-ipv4-ippool"]`) +- `ipv6_pools`: An array of CIDR strings or pool names. (e.g., `"ipv6_pools": ["2001:db8::1/120", "namedpool"]`) + +Example CNI config: + +```json +{ + "name": "any_name", + "cniVersion": "0.1.0", + "type": "calico", + "ipam": { + "type": "calico-ipam", + "assign_ipv4": "true", + "assign_ipv6": "true", + "ipv4_pools": ["10.0.0.0/24", "20.0.0.0/16", "default-ipv4-ippool"], + "ipv6_pools": ["2001:db8::1/120", "default-ipv6-ippool"] + } +} +``` + +:::note + +`ipv6_pools` will be respected only when `assign_ipv6` is set to `"true"`. + +::: + +Any IP pools specified in the CNI config must have already been created. It is an error to specify IP pools in the config that do not exist. + +### Container settings + +The following options allow configuration of settings within the container namespace. + +- allow_ip_forwarding (default is `false`) + +```json +{ + "name": "any_name", + "cniVersion": "0.1.0", + "type": "calico", + "ipam": { + "type": "calico-ipam" + }, + "container_settings": { + "allow_ip_forwarding": true + } +} +``` + +### Readiness Gates + +The following option makes CNI plugin wait for specified endpoint(s) to be ready before configuring pod networking. + +- `readiness_gates` + +This is an optional property that takes an array of URLs. Each URL specified will be polled for readiness and pod networking will continue startup once all readiness_gates are ready. + +Example CNI config: + +```json +{ + "name": "any_name", + "cniVersion": "0.1.0", + "type": "calico", + "ipam": { + "type": "calico-ipam" + }, + "readiness_gates": ["http://localhost:9099/readiness", "http://localhost:8888/status"] +} +``` + +## Kubernetes specific + +When using the $[prodname] CNI plugin with Kubernetes, the plugin must be able to access the Kubernetes API server to find the labels assigned to the Kubernetes pods. The recommended way to configure access is through a `kubeconfig` file specified in the `kubernetes` section of the network config. e.g. + +```json +{ + "name": "any_name", + "cniVersion": "0.1.0", + "type": "calico", + "kubernetes": { + "kubeconfig": "/path/to/kubeconfig" + }, + "ipam": { + "type": "calico-ipam" + } +} +``` + +As a convenience, the API location can also be configured directly, e.g. + +```json +{ + "name": "any_name", + "cniVersion": "0.1.0", + "type": "calico", + "kubernetes": { + "k8s_api_root": "http://127.0.0.1:8080" + }, + "ipam": { + "type": "calico-ipam" + } +} +``` + +### Enabling Kubernetes policy + +If you wish to use the Kubernetes `NetworkPolicy` resource then you must set a policy type in the network config. +There is a single supported policy type, `k8s`. When set, +you must also run `$[imageNames.kubeControllers]` with the policy, profile, and workloadendpoint controllers enabled. + +```json +{ + "name": "any_name", + "cniVersion": "0.1.0", + "type": "calico", + "policy": { + "type": "k8s" + }, + "kubernetes": { + "kubeconfig": "/path/to/kubeconfig" + }, + "ipam": { + "type": "calico-ipam" + } +} +``` + +When using `type: k8s`, the $[prodname] CNI plugin requires read-only Kubernetes API access to the `Pods` resource in all namespaces. + +## IPAM + +### Using host-local IPAM + +Calico can be configured to use [host-local IPAM](https://www.cni.dev/plugins/current/ipam/host-local/) instead of the default `calico-ipam`. Host +local IPAM uses a pre-determined CIDR per-host, and stores allocations locally on each node. This is in contrast to Calico IPAM, which dynamically +allocates blocks of addresses and single addresses alike in response to cluster needs. + +Host local IPAM is generally only used on clusters where integration with the Kubernetes [route controller](https://kubernetes.io/docs/concepts/architecture/cloud-controller/#route-controller) is necessary. +Note that some Calico features - such as the ability to request a specific address or pool for a pod - require Calico IPAM to function, and will not work with host-local IPAM enabled. + + + + +The `host-local` IPAM plugin can be configured by setting the `Spec.CNI.IPAM.Plugin` field to `HostLocal` on the [operator.tigera.io/Installation](../installation/api.mdx#installation) API. + +Calico will use the `host-local` IPAM plugin to allocate IPv4 addresses from the node's IPv4 pod CIDR if there is an IPv4 pool configured in `Spec.IPPools`, and an IPv6 address from the node's IPv6 pod CIDR if +there is an IPv6 pool configured in `Spec.IPPools`. + +The following example configures Calico to assign dual-stack IPs to pods using the host-local IPAM plugin. + +```yaml +kind: Installation +apiVersion: operator.tigera.io/v1 +metadata: + name: default +spec: + calicoNetwork: + ipPools: + - cidr: 192.168.0.0/16 + - cidr: 2001:db8::/64 + cni: + type: Calico + ipam: + type: HostLocal +``` + + + + +When using the CNI `host-local` IPAM plugin, two special values - `usePodCidr` and `usePodCidrIPv6` - are allowed for the subnet field (either at the top-level, or in a "range"). This tells the plugin to determine the subnet to use from the Kubernetes API based on the Node.podCIDR field. $[prodname] does not use the `gateway` field of a range so that field is not required and it will be ignored if present. + +:::note + +`usePodCidr` and `usePodCidrIPv6` can only be used as the value of the `subnet` field, it cannot be used in +`rangeStart` or `rangeEnd` so those values are not useful if `subnet` is set to `usePodCidr`. + +::: + +$[prodname] supports the host-local IPAM plugin's `routes` field as follows: + +- If there is no `routes` field, $[prodname] will install a default `0.0.0.0/0`, and/or `::/0` route into the pod (depending on whether the pod has an IPv4 and/or IPv6 address). + +- If there is a `routes` field then $[prodname] will program _only_ the routes in the routes field into the pod. Since $[prodname] implements a point-to-point link into the pod, the `gw` field is not required and it will be ignored if present. All routes that $[prodname] installs will have $[prodname]'s link-local IP as the next hop. + +$[prodname] CNI plugin configuration: + +- `node_name` + - The node name to use when looking up the CIDR value (defaults to current hostname) + +```json +{ + "name": "any_name", + "cniVersion": "0.1.0", + "type": "calico", + "kubernetes": { + "kubeconfig": "/path/to/kubeconfig", + "node_name": "node-name-in-k8s" + }, + "ipam": { + "type": "host-local", + "ranges": [[{ "subnet": "usePodCidr" }], [{ "subnet": "usePodCidrIPv6" }]], + "routes": [{ "dst": "0.0.0.0/0" }, { "dst": "2001:db8::/96" }] + } +} +``` + +When making use of the `usePodCidr` or `usePodCidrIPv6` options, the $[prodname] CNI plugin requires read-only Kubernetes API access to the `Nodes` resource. + +#### Configuring node and typha + +When using `host-local` IPAM with the Kubernetes API datastore, you must configure both $[nodecontainer] and the Typha deployment to use the `Node.podCIDR` field by setting the environment variable `USE_POD_CIDR=true` in each. + + + + +### Using Kubernetes annotations + +#### Specifying IP pools on a per-namespace or per-pod basis + +In addition to specifying IP pools in the CNI config as discussed above, $[prodname] IPAM supports specifying IP pools per-namespace or per-pod using the following [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/). + +- `cni.projectcalico.org/ipv4pools`: A list of configured IPv4 Pools from which to choose an address for the pod. + + Example: + + ```yaml + annotations: + 'cni.projectcalico.org/ipv4pools': '["default-ipv4-ippool"]' + ``` + +- `cni.projectcalico.org/ipv6pools`: A list of configured IPv6 Pools from which to choose an address for the pod. + + Example: + + ```yaml + annotations: + 'cni.projectcalico.org/ipv6pools': '["2001:db8::1/120"]' + ``` + +If provided, these IP pools will override any IP pools specified in the CNI config. + +:::note + +This requires the IP pools to exist before `ipv4pools` or +`ipv6pools` annotations are used. Requesting a subset of an IP pool +is not supported. IP pools requested in the annotations must exactly +match a configured [IPPool](../resources/ippool.mdx) resource. + +::: + +:::note + +The $[prodname] CNI plugin supports specifying an annotation per namespace. +If both the namespace and the pod have this annotation, the pod information will be used. +Otherwise, if only the namespace has the annotation the annotation of the namespace will +be used for each pod in it. + +::: + +#### Requesting a specific IP address + +You can also request a specific IP address through [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) with $[prodname] IPAM. +There are two annotations to request a specific IP address: + +- `cni.projectcalico.org/ipAddrs`: A list of IPv4 and/or IPv6 addresses to assign to the Pod. The requested IP addresses will be assigned from $[prodname] IPAM and must exist within a configured IP pool. + + Example: + + ```yaml + annotations: + 'cni.projectcalico.org/ipAddrs': '["192.168.0.1"]' + ``` + +- `cni.projectcalico.org/ipAddrsNoIpam`: A list of IPv4 and/or IPv6 addresses to assign to the Pod, bypassing IPAM. Any IP conflicts and routing have to be taken care of manually or by some other system. + $[prodname] will only distribute routes to a Pod if its IP address falls within a $[prodname] IP pool using BGP mode. Calico will not distribute ipAddrsNoIpam routes when operating in VXLAN mode. If you assign an IP address that is not in a $[prodname] IP pool or if its IP address falls within a $[prodname] IP pool that uses VXLAN encapsulation, you must ensure that routing to that IP address is taken care of through another mechanism. + + Example: + + ```yaml + annotations: + 'cni.projectcalico.org/ipAddrsNoIpam': '["10.0.0.1"]' + ``` + + The ipAddrsNoIpam feature is disabled by default. It can be enabled in the feature_control section of the CNI network config: + + ```json + { + "name": "any_name", + "cniVersion": "0.1.0", + "type": "calico", + "ipam": { + "type": "calico-ipam" + }, + "feature_control": { + "ip_addrs_no_ipam": true + } + } + ``` + + :::caution + + This feature allows for the bypassing of network policy via IP spoofing. + Users should make sure the proper admission control is in place to prevent users from selecting arbitrary IP addresses. + + ::: + +:::note + +- The `ipAddrs` and `ipAddrsNoIpam` annotations can't be used together. +- You can only specify one IPv4/IPv6 or one IPv4 and one IPv6 address with these annotations. +- When `ipAddrs` or `ipAddrsNoIpam` is used with `ipv4pools` or `ipv6pools`, `ipAddrs` / `ipAddrsNoIpam` take priority. + +::: + +#### Requesting a floating IP + +You can request a floating IP address for a pod through [Kubernetes annotations](https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/) with $[prodname]. + +:::note + +The specified address must belong to an IP Pool for advertisement to work properly. + +::: + +- `cni.projectcalico.org/floatingIPs`: A list of floating IPs which will be assigned to the pod's workload endpoint. + + Example: + + ```yaml + annotations: + 'cni.projectcalico.org/floatingIPs': '["10.0.0.1"]' + ``` + + The floatingIPs feature is disabled by default. It can be enabled in the feature_control section of the CNI network config: + + ```json + { + "name": "any_name", + "cniVersion": "0.1.0", + "type": "calico", + "ipam": { + "type": "calico-ipam" + }, + "feature_control": { + "floating_ips": true + } + } + ``` + + :::caution + + This feature can allow pods to receive traffic which may not have been intended for that pod. + Users should make sure the proper admission control is in place to prevent users from selecting arbitrary floating IP addresses. + + ::: + +### Using IP pools node selectors + +Nodes will only assign workload addresses from IP pools which select them. By +default, IP pools select all nodes, but this can be configured using the +`nodeSelector` field. Check out the [IP pool resource document](../resources/ippool.mdx) +for more details. + +Example: + +1. Create (or update) an IP pool that only allocates IPs for nodes where it + contains a label `rack=0`. + + ```bash + kubectl create -f -< + + +[installation]: /reference/installation/api.mdx diff --git a/calico-cloud_versioned_docs/version-24-1/reference/component-resources/configure-resources.mdx b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/configure-resources.mdx new file mode 100644 index 0000000000..f0882909e1 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/configure-resources.mdx @@ -0,0 +1,699 @@ +--- +description: Reference for setting Kubernetes resource requests and limits on Calico Cloud components managed by the Tigera Operator in connected clusters. +--- + +# Configure resource requests and limits + +## Big picture + +Resource requests and limits are essential configurations for managing resource allocation and ensuring optimal performance of Kubernetes workloads. In $[prodname], these configurations can be customized using custom resources to meet specific requirements and optimize resource utilization. + +:::note +It's important to note that the CPU and memory values used in the examples are for demonstration purposes and should be adjusted based on individual system requirements. To find the list of all applicable containers for a component, please refer to its specification. +::: + +## APIServer custom resource + +The [APIServer](../../reference/installation/api.mdx#apiserver) CR provides a way to configure APIServerDeployment. The following sections provide example configurations for this CR. + +### APIServerDeployment + +To configure resource specification for the [APIServerDeployment](../../reference/installation/api.mdx#apiserverdeployment), patch the installation CR using the below command: + +```bash +$ kubectl patch apiserver tigera-secure --type=merge --patch='{"spec": {"apiServerDeployment":{"spec": {"template": {"spec": {"containers":[{"name":"calico-apiserver","resources":{"limits":{"cpu":"1", "memory":"1000Mi"},"requests":{"cpu":"100m", "memory":"100Mi"}}},{"name":"tigera-queryserver","resources":{"limits":{"cpu":"1", "memory":"1000Mi"},"requests":{"cpu":"100m", "memory":"100Mi"}}}]}}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +$ kubectl get deployment.apps/calico-apiserver -n calico-system -o json | jq '.spec.template.spec.containers[] | {name: .name, resources: .resources}' +``` + +This command will output the configured resource requests and limits for the Calico APIServerDeployment component in JSON format. + +```bash +{ + "name": "calico-apiserver", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +{ + "name": "tigera-queryserver", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +``` + +## ApplicationLayer custom resource + +The [ApplicationLayer](../../reference/installation/api.mdx#applicationlayer) CR provides a way to configure resources for L7LogCollectorDaemonSet. The following sections provide example configurations for this CR. + +### L7LogCollectorDaemonSet + +To configure resource specification for the [L7LogCollectorDaemonSet](../../reference/installation/api.mdx#l7logcollectordaemonset), patch the ApplicationLayer CR using the below command: + +```bash +$ kubectl patch applicationlayer tigera-secure --type=merge --patch='{"spec": {"l7LogCollectorDaemonSet":{"spec": {"template": {"spec": {"containers":[{"name":"l7-collector","resources":{"limits":{"cpu":"1", "memory":"1000Mi"},"requests":{"cpu":"100m", "memory":"100Mi"}}},{"name":"envoy-proxy","resources":{"limits":{"cpu":"1", "memory":"1000Mi"},"requests":{"cpu":"100m", "memory":"100Mi"}}}]}}}}}}' +applicationlayer.operator.tigera.io/tigera-secure patched +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +$ kubectl get daemonset.apps/l7-log-collector -n calico-system -o json | jq '.spec.template.spec.containers[] | {name: .name, resources: .resources}' +``` + +This command will output the configured resource requests and limits for the Calico L7LogCollectorDaemonSet component in JSON format. + +```bash +{ + "name": "envoy-proxy", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +{ + "name": "l7-collector", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +``` + +## Compliance custom resource + +The Compliance CR provides a way to configure resources for ComplianceControllerDeployment, ComplianceSnapshotterDeployment, ComplianceBenchmarkerDaemonSet, ComplianceServerDeployment, ComplianceReporterPodTemplate. The following sections provide example configurations for this CR. + +### ComplianceControllerDeployment + +To configure resource specification for the ComplianceControllerDeployment, patch the Compliance CR using the below command: + +```bash +kubectl patch compliance tigera-secure --type=merge --patch='{"spec": {"complianceControllerDeployment":{"spec": {"template": {"spec": {"containers":[{"name":"compliance-controller","resources":{"limits":{"cpu":"1", "memory":"1000Mi"},"requests":{"cpu":"100m", "memory":"100Mi"}}}]}}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +kubectl get deployment.apps/compliance-controller -n tigera-compliance -o json|jq '.spec.template.spec.containers[] | {name: .name, resources: .resources}' +``` + +This command will output the configured resource requests and limits for the ComplianceControllerDeployment component in JSON format. + +```bash +{ + "name": "compliance-controller", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +``` + + +### ComplianceSnapshotterDeployment + +To configure resource specification for the ComplianceSnapshotterDeployment, patch the Compliance CR using the below command: + +```bash +kubectl patch compliance tigera-secure --type=merge --patch='{"spec": {"complianceSnapshotterDeployment":{"spec": {"template": {"spec": {"containers":[{"name":"compliance-snapshotter","resources":{"limits":{"cpu":"1", "memory":"1000Mi"},"requests":{"cpu":"100m", "memory":"100Mi"}}}]}}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +kubectl get deployment.apps/compliance-snapshotter -n tigera-compliance -o json|jq '.spec.template.spec.containers[] | {name: .name, resources: .resources}' +``` + +This command will output the configured resource requests and limits for the ComplianceSnapshotterDeployment in JSON format. + +```bash +{ + "name": "compliance-snapshotter", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +``` + + +### ComplianceBenchmarkerDaemonSet + +To configure resource specification for the ComplianceBenchmarkerDaemonSet, patch the Compliance CR using the below command: + +```bash +kubectl patch compliance tigera-secure --type=merge --patch='{"spec": {"complianceBenchmarkerDaemonSet":{"spec": {"template": {"spec": {"containers":[{"name":"compliance-benchmarker","resources":{"limits":{"cpu":"1", "memory":"1000Mi"},"requests":{"cpu":"100m", "memory":"100Mi"}}}]}}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +kubectl get daemonset.apps/compliance-benchmarker -n tigera-compliance -o json |jq '.spec.template.spec.containers[] | {name: .name, resources: .resources}' +``` + +```bash +{ + "name": "compliance-benchmarker", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +``` + +This command will output the configured resource requests and limits for the ComplianceBenchmarkerDaemonSet in JSON format. + +### ComplianceServerDeployment + +To configure resource specification for the ComplianceServerDeployment, patch the Compliance CR using the below command: + +```bash +kubectl patch compliance tigera-secure --type=merge --patch='{"spec": {"complianceServerDeployment":{"spec": {"template": {"spec": {"containers":[{"name":"compliance-server","resources":{"limits":{"cpu":"1", "memory":"1000Mi"},"requests":{"cpu":"100m", "memory":"100Mi"}}}]}}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +kubectl get deployment.apps/compliance-server -n tigera-compliance -o json| jq '.spec.template.spec.containers[] | {name: .name, resources: .resources}' +``` + +This command will output the configured resource requests and limits for the ComplianceServerDeployment in JSON format. + +```bash +{ + "name": "compliance-server", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +``` + + +### ComplianceReporterPodTemplate. + +To configure resource specification for the ComplianceReporterPodTemplate, patch the Compliance CR using the below command: + +```bash +kubectl patch compliance tigera-secure --type=merge --patch='{"spec": {"complianceReporterPodTemplate": {"template": {"spec": {"containers":[{"name":"reporter","resources":{"limits":{"cpu":"1", "memory":"1000Mi"},"requests":{"cpu":"100m", "memory":"100Mi"}}}]}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +kubectl get Podtemplates tigera.io.report -n tigera-compliance -o json | jq '.template.spec.containers[] | {name: .name, resources: .resources}' +``` + +This command will output the configured resource requests and limits for the ComplianceReporterPodTemplate component in JSON format. + +```bash +{ + "name": "reporter", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +``` + +## Installation custom resource + +The [Installation CR](../../reference/installation/api.mdx) provides a way to configure resources for various Calico Enterprise components, including TyphaDeployment, calicoNodeDaemonSet, CalicoNodeWindowsDaemonSet, csiNodeDriverDaemonSet and KubeControllersDeployment. The following sections provide example configurations for this CR. + +Example Configurations: + + +### TyphaDeployment + +To configure resource specification for the [TyphaDeployment](../../reference/installation/api.mdx#typhadeployment), patch the installation CR using the below command: + +```bash +kubectl patch installations default --type=merge --patch='{"spec": {"typhaDeployment": {"spec": {"template": {"spec": {"containers": [{"name": "calico-typha", "resources": {"requests": {"cpu": "100m", "memory": "100Mi"}, "limits": {"cpu": "1", "memory": "1000Mi"}}}]}}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +$ kubectl get deployment.apps/calico-typha -n calico-system -o json | jq '.spec.template.spec.containers[]| {name:.name,resources:.resources}' +``` + +This command will output the configured resource requests and limits for the Calico TyphaDeployment component in JSON format. + +```bash +{ + "name": "calico-typha", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +``` + + +### CalicoNodeDaemonSet + +To configure resource requests for the [calicoNodeDaemonSet](../../reference/installation/api.mdx#caliconodedaemonset) component, patch the installation CR using the below command: + +```bash +$ kubectl patch installations default --type=merge --patch='{"spec": {"calicoNodeDaemonSet":{"spec": {"template": {"spec": {"containers":[{"name":"calico-node","resources":{"requests":{"cpu":"100m", "memory":"100Mi"}, "limits":{"cpu":"1", "memory":"1000Mi"}}}]}}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +$ kubectl get daemonset.apps/calico-node -n calico-system -o json | jq '.spec.template.spec.containers[]| {name:.name,resources:.resources}' +``` + +This command will output the configured resource requests and limits for the Calico calicoNodeDaemonSet component in JSON format. + +```bash +{ + "name": "calico-node", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +``` + +### calicoNodeWindowsDaemonSet + +To configure resource requests for the [calicoNodeWindowsDaemonSet](../../reference/installation/api.mdx#caliconodewindowsdaemonset) component, patch the installation CR using the below command: + +```bash +$ kubectl patch installations default --type=merge --patch='{"spec": {"calicoNodeWindowsDaemonSet":{"spec": {"template": {"spec": {"containers":[{"name":"calico-node-windows","resources":{"requests":{"cpu":"100m", "memory":"100Mi"}, "limits":{"cpu":"1", "memory":"1000Mi"}}}]}}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +$ kubectl get daemonset.apps/calico-node -n calico-system -o json | jq '.spec.template.spec.containers[]| {name:.name,resources:.resources}' +``` + +This command will output the configured resource requests and limits for the Calico calicoNodeWindowsDaemonSet component in JSON format. + +```bash +{ + "name": "calico-node", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +``` + +### CalicoKubeControllersDeployment + +To configure resource requests for the [CalicoKubeControllersDeployment](../../reference/installation/api.mdx#calicokubecontrollersdeployment) component, patch the installation CR using the below command: + +```bash +$ kubectl patch installations default --type=merge --patch='{"spec": {"calicoKubeControllersDeployment":{"spec": {"template": {"spec": {"containers":[{"name":"calico-kube-controllers","resources":{"requests":{"cpu":"100m", "memory":"100Mi"}, "limits":{"cpu":"1", "memory":"1000Mi"}}}]}}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +$ kubectl get deployment.apps/calico-kube-controllers -n calico-system -o json | jq '.spec.template.spec.containers[]| {name:.name,resources:.resources}' +``` + +This command will output the configured resource requests and limits for the Calico CalicoKubeControllersDeployment component in JSON format. + +```bash +{ + "name": "calico-kube-controllers", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} + +``` + +### CSINodeDriverDaemonSet + +To configure resource requests for the [CSINodeDriverDaemonSet](../../reference/installation/api.mdx#csinodedriverdaemonset) component, patch the installation CR using the below command: + +```bash +$ kubectl patch installations default --type=merge --patch='{"spec": {"csiNodeDriverDaemonSet":{"spec": {"template": {"spec": {"containers":[{"name":"calico-csi","resources":{"requests":{"cpu":"100m", "memory":"100Mi"}, "limits":{"cpu":"1", "memory":"1000Mi"}}},{"name":"csi-node-driver-registrar","resources":{"requests":{"cpu":"50m", "memory":"50Mi"}, "limits":{"cpu":"1", "memory":"1000Mi"}}}]}}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +$ kubectl get daemonset.apps/csi-node-driver -n calico-system -o json | jq '.spec.template.spec.containers[]| {name:.name,resources:.resources}' +``` + +This command will output the configured resource requests and limits for the Calico calicoNodeDaemonSet component in JSON format. + +```bash +{ + "name": "calico-csi", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +{ + "name": "csi-node-driver-registrar", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "50m", + "memory": "50Mi" + } + } +} +``` + +## IntrusionDetection custom resource + +The [IntrusionDetection](../../reference/installation/api.mdx#intrusiondetection) CR provides a way to configure resources for IntrusionDetectionControllerDeployment. The following sections provide example configurations for this CR. + +### IntrusionDetectionControllerDeployment. + +To configure resource specification for the [IntrusionDetectionControllerDeployment](../../reference/installation/api.mdx#intrusiondetectioncontrollerdeployment), patch the IntrusionDetection CR using the below command: + +```bash +$ kubectl patch intrusiondetection tigera-secure --type=merge --patch='{"spec": {"intrusionDetectionControllerDeployment":{"spec": {"template": {"spec": {"containers":[{"name":"webhooks-processor","resources":{"limits":{"cpu":"1", "memory":"1000Mi"},"requests":{"cpu":"100m", "memory":"1000Mi"}}},{"name":"controller","resources":{"limits":{"cpu":"1", "memory":"1000Mi"},"requests":{"cpu":"100m", "memory":"1000Mi"}}}]}}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +$ kubectl get deployment.apps/intrusion-detection-controller -n tigera-intrusion-detection -o json|jq '.spec.template.spec.containers[] | {name: .name, resources: .resources}' +``` + +This command will output the configured resource requests and limits for the IntrusionDetectionControllerDeployment in JSON format. + +```bash +{ + "name": "controller", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "1000Mi" + } + } +} +{ + "name": "webhooks-processor", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "1000Mi" + } + } +} +``` + +## LogCollector custom resource + +The [LogCollector](../../reference/installation/api.mdx#logcollector) CR provides a way to configure resources for FluentdDaemonSet, EKSLogForwarderDeployment. + +### FluentdDaemonSet. + +To configure resource specification for the [FluentdDaemonSet](../../reference/installation/api.mdx#fluentddaemonset), patch the LogCollector CR using the below command: + +```bash +kubectl patch logcollector tigera-secure --type=merge --patch='{"spec": {"fluentdDaemonSet":{"spec": {"template": {"spec": {"containers":[{"name":"fluentd","resources":{"limits":{"cpu":"1", "memory":"1000Mi"},"requests":{"cpu":"100m", "memory":"100Mi"}}}]}}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +kubectl get daemonset.apps/fluentd-node -n tigera-fluentd -o json | jq '.spec.template.spec.containers[] | {name: .name, resources: .resources}' +``` + +This command will output the configured resource requests and limits for the FluentdDaemonSet in JSON format. + +```bash +{ + "name": "fluentd", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +``` + + +### EKSLogForwarderDeployment. + +To configure resource specification for the [EKSLogForwarderDeployment](../../reference/installation/api.mdx#ekslogforwarderdeployment), patch the LogCollector CR using the below command: + +```bash +kubectl patch logcollector tigera-secure --type=merge --patch='{"spec": {"eksLogForwarderDeployment": {"spec": {"template": {"spec": {"containers":[{"name":"eks-log-forwarder","resources":{"limits":{"cpu":"1", "memory":"1000Mi"},"requests":{"cpu":"100m", "memory":"100Mi"}}}]}}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +kubectl get deployment.apps/eks-log-forwarder -n tigera-fluentd -o json | jq '.spec.template.spec.containers[] | {name: .name, resources: .resources}' +``` + +This command will output the configured resource requests and limits for the EKSLogForwarderDeployment in JSON format. + +```bash +{ + "name": "eks-log-forwarder", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +``` + +## ManagementClusterConnection custom resource + +The [ManagementClusterConnection](../../reference/installation/api.mdx#managementclusterconnection) CR provides a way to configure resources for GuardianDeployment. The following sections provide example configurations for this CR. + +### GuardianDeployment. + +To configure resource specification for the [GuardianDeployment](../../reference/installation/api.mdx#guardiandeployment), patch the ManagementClusterConnection CR using the below command: + +```bash +kubectl patch managementclusterconnection tigera-secure --type=merge --patch='{"spec": {"guardianDeployment":{"spec": {"template": {"spec": {"containers":[{"name":"tigera-guardian","resources":{"limits":{"cpu":"1", "memory":"1000Mi"},"requests":{"cpu":"100m", "memory":"100Mi"}}}]}}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +kubectl get deployment.apps/tigera-guardian -n tigera-guardian -o json | jq '.spec.template.spec.containers[] | {name: .name, resources: .resources}' +``` + +This command will output the configured resource requests and limits for the GuardianDeployment in JSON format. + +```bash +{ + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } +} +``` + +## PacketCaptureAPI custom resource + +The [PacketCaptureAPI](../../reference/installation/api.mdx#packetcaptureapi) CR provides a way to configure resources for PacketCapture. The following sections provide example configurations for this CR. + +### PacketCaptureAPIDeployment + +To configure resource specification for the [PacketCaptureAPI](../../reference/installation/api.mdx#packetcaptureapi), patch the PacketCapture CR using the below command: + +```bash +kubectl patch packetcaptureapis tigera-secure --type=merge --patch='{"spec": {"packetCaptureAPIDeployment":{"spec": {"template": {"spec": {"containers":[{"name":"tigera-packetcapture-server","resources":{"limits":{"cpu":"1", "memory":"1000Mi"},"requests":{"cpu":"100m", "memory":"100Mi"}}}]}}}}}}' +``` +This command sets the CPU request to 100 milliCPU (mCPU) and the memory request is set to 100 Mebibytes (MiB) while the CPU limit is set to 1 CPU and the memory limit is set to 1000 Mebibytes (MiB). + +#### Verification + +You can verify the configured resources using the following command: + +```bash +kubectl get deployment.apps/tigera-packetcapture -n tigera-packetcapture -o json | jq '.spec.template.spec.containers[] | {name: .name, resources: .resources}' +``` + +This command will output the configured resource requests and limits for the PacketCaptureDeployment in JSON format. + +```bash +{ + "name": "tigera-packetcapture-server", + "resources": { + "limits": { + "cpu": "1", + "memory": "1000Mi" + }, + "requests": { + "cpu": "100m", + "memory": "100Mi" + } + } +} +``` diff --git a/calico-cloud_versioned_docs/version-24-1/reference/component-resources/index.mdx b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/index.mdx new file mode 100644 index 0000000000..939169a0e0 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/index.mdx @@ -0,0 +1,11 @@ +--- +description: Reference content for Calico Cloud component resources covering CNI configuration, kube-controllers, and node components in connected clusters. +hide_table_of_contents: true +--- + +# Component resources + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/reference/component-resources/kube-controllers/index.mdx b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/kube-controllers/index.mdx new file mode 100644 index 0000000000..c42646d765 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/kube-controllers/index.mdx @@ -0,0 +1,11 @@ +--- +description: kube-controllers is a set of Kubernetes controllers for Calico Cloud. +hide_table_of_contents: true +--- + +# kube-controllers + +The `$[imageNames.kubeControllers]` container runs a set of controllers that monitor the Kubernetes API and reconcile cluster state with the $[prodname] datastore. + +- To configure the controllers, see the [KubeControllersConfiguration](../../resources/kubecontrollersconfig.mdx) resource reference. +- For exported Prometheus metrics, see [Prometheus statistics](prometheus.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/reference/component-resources/kube-controllers/prometheus.mdx b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/kube-controllers/prometheus.mdx new file mode 100644 index 0000000000..e5fbee5ac8 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/kube-controllers/prometheus.mdx @@ -0,0 +1,86 @@ +--- +description: Prometheus metrics reference for the kube-controllers component in Calico Cloud connected clusters covering reconcile latency and queue depth. +--- + +# Monitoring kube-controllers with Prometheus + +kube-controllers can be configured to report a number of metrics through Prometheus. This reporting is enabled by default on port 9094. See the +[configuration reference](../../resources/kubecontrollersconfig.mdx) for how to change metrics reporting configuration (or disable it completely). + +## Metric reference + +#### kube-controllers specific + +kube-controllers exports a number of Prometheus metrics. The current set is as follows. Since some metrics +may be tied to particular implementation choices inside kube-controllers we can't make any hard guarantees that +metrics will persist across releases. However, we aim not to make any spurious changes to +existing metrics. + +| Metric Name | Labels | Description | +| ------------------------------------ | ------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `ipam_allocations_in_use` | ippool, node | Number of Calico IP allocations currently in use by a workload or interface. | +| `ipam_allocations_borrowed` | ippool, node | Number of Calico IP allocations currently in use where the allocation was borrowed from a block affine to another node. | +| `ipam_allocations_gc_candidates` | ippool, node | Number of Calico IP allocations currently marked by the GC as potential leaks. This metric returns to zero under normal GC operation. | +| `ipam_allocations_gc_reclamations` | ippool, node | Count of Calico IP allocations that have been reclaimed by the GC. Increase of this counter corresponds with a decrease of the candidates gauge under normal operation. | +| `ipam_blocks` | ippool, node | Number of IPAM blocks. | +| `ipam_ippool_size` | ippool | Number of IP addresses in the IP Pool CIDR. | +| `ipam_blocks_per_node` | node | Number of IPAM blocks, indexed by the node to which they have affinity. Prefer `ipam_blocks` for new integrations. | +| `ipam_allocations_per_node` | node | Number of Calico IP allocations, indexed by node on which the allocation was made. Prefer `ipam_allocations_in_use` for new integrations. | +| `ipam_allocations_borrowed_per_node` | node | Number of Calico IP allocations borrowed from a non-affine block, indexed by node on which the allocation was made. Prefer `ipam_allocations_borrowed` for new integrations. | +| `remote_cluster_connection_status` | remote_cluster_name | Status of the remote cluster connection in federation. Represented as numeric values 0 (NotConnecting) ,1 (Connecting), 2 (InSync), 3 (ReSyncInProgress), 4 (ConfigChangeRestartRequired), 5 (ConfigInComplete). | + +Labels can be interpreted as follows: + +| Label Name | Description | +| --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `node` | For allocation metrics, the node on which the allocation was made. For block metrics, the node for which the block has affinity. If the block has no affinity, value will be `no_affinity`. | +| `ippool` | The IP Pool that the IPAM block occupies. If there is no IP Pool which matches the block, value will be `no_ippool`. | +| `remote_cluster_name` | Name of the remote cluster in federation. | + +Prometheus metrics are self-documenting, with metrics turned on, `curl` can be used to list the +metrics along with their help text and type information. + +```bash +curl -s http://localhost:9094/metrics | head +``` + +#### CPU / memory metrics + +kube-controllers also exports the default set of metrics that Prometheus makes available. Currently, those +include: + +| Name | Description | +| -------------------------------------------- | ------------------------------------------------------------------ | +| `go_gc_duration_seconds` | A summary of the GC invocation durations. | +| `go_goroutines` | Number of goroutines that currently exist. | +| `go_memstats_alloc_bytes` | Number of bytes allocated and still in use. | +| `go_memstats_alloc_bytes_total` | Total number of bytes allocated, even if freed. | +| `go_memstats_buck_hash_sys_bytes` | Number of bytes used by the profiling bucket hash table. | +| `go_memstats_frees_total` | Total number of frees. | +| `go_memstats_gc_sys_bytes` | Number of bytes used for garbage collection system metadata. | +| `go_memstats_heap_alloc_bytes` | Number of heap bytes allocated and still in use. | +| `go_memstats_heap_idle_bytes` | Number of heap bytes waiting to be used. | +| `go_memstats_heap_inuse_bytes` | Number of heap bytes that are in use. | +| `go_memstats_heap_objects` | Number of allocated objects. | +| `go_memstats_heap_released_bytes_total` | Total number of heap bytes released to OS. | +| `go_memstats_heap_sys_bytes` | Number of heap bytes obtained from system. | +| `go_memstats_last_gc_time_seconds` | Number of seconds since 1970 of last garbage collection. | +| `go_memstats_lookups_total` | Total number of pointer lookups. | +| `go_memstats_mallocs_total` | Total number of mallocs. | +| `go_memstats_mcache_inuse_bytes` | Number of bytes in use by mcache structures. | +| `go_memstats_mcache_sys_bytes` | Number of bytes used for mcache structures obtained from system. | +| `go_memstats_mspan_inuse_bytes` | Number of bytes in use by mspan structures. | +| `go_memstats_mspan_sys_bytes` | Number of bytes used for mspan structures obtained from system. | +| `go_memstats_next_gc_bytes` | Number of heap bytes when next garbage collection will take place. | +| `go_memstats_other_sys_bytes` | Number of bytes used for other system allocations. | +| `go_memstats_stack_inuse_bytes` | Number of bytes in use by the stack allocator. | +| `go_memstats_stack_sys_bytes` | Number of bytes obtained from system for stack allocator. | +| `go_memstats_sys_bytes` | Number of bytes obtained by system. Sum of all system allocations. | +| `process_cpu_seconds_total` | Total user and system CPU time spent in seconds. | +| `process_max_fds` | Maximum number of open file descriptors. | +| `process_open_fds` | Number of open file descriptors. | +| `process_resident_memory_bytes` | Resident memory size in bytes. | +| `process_start_time_seconds` | Start time of the process since Unix epoch in seconds. | +| `process_virtual_memory_bytes` | Virtual memory size in bytes. | +| `promhttp_metric_handler_requests_in_flight` | Current number of scrapes being served. | +| `promhttp_metric_handler_requests_total` | Total number of scrapes by HTTP status code. | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/component-resources/node/configuration.mdx b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/node/configuration.mdx new file mode 100644 index 0000000000..5727001739 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/node/configuration.mdx @@ -0,0 +1,313 @@ +--- +description: Reference for configuring the cnx-node container in Calico Cloud connected clusters through environment variables that control Felix, BIRD, and confd. +--- + +# Configuring cnx-node + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +The `$[nodecontainer]` container is deployed to every node (on Kubernetes, by a DaemonSet), and runs three internal daemons: + +- Felix, the Calico daemon that runs on every node and provides endpoints. +- BIRD, the BGP daemon that distributes routing information to other nodes. +- confd, a daemon that watches the Calico datastore for config changes and updates BIRD’s config files. + +For manifest-based installations, `$[nodecontainer]` is primarily configured through environment +variables, typically set in the deployment manifest. Individual nodes may also be updated through the Node +custom resource. `$[nodecontainer]` can also be configured through the Calico Operator. + +The rest of this page lists the available configuration options, and is followed by specific considerations for +various settings. + + + + +`$[nodecontainer]` does not need to be configured directly when installed by the operator. For a complete operator +configuration reference, see [the installation API reference documentation][installation]. + + + + +## Environment variables + +### Configuring the default IP pool(s) + +Calico uses IP pools to configure how addresses are allocated to pods, and how networking works for certain +sets of addresses. You can see the full schema for IP pools here. + +`$[nodecontainer]` can be configured to create a default IP pool for you, but only if none already +exist in the cluster. The following options control the parameters on the created pool. + +| Environment | Description | Schema | +| ---------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | +| CALICO_IPV4POOL_CIDR | The IPv4 Pool to create if none exists at start up. It is invalid to define this variable and NO_DEFAULT_POOLS. [Default: First not used in locally of (192.168.0.0/16, 172.16.0.0/16, .., 172.31.0.0/16) ] | IPv4 CIDR | +| CALICO_IPV4POOL_BLOCK_SIZE | Block size to use for the IPv4 Pool created at startup. Block size for IPv4 should be in the range 20-32 (inclusive) [Default: `26`] | int | +| CALICO_IPV4POOL_IPIP | IPIP Mode to use for the IPv4 Pool created at start up. If set to a value other than `Never`, `CALICO_IPV4POOL_VXLAN` should not be set. [Default: `Always`] | Always, CrossSubnet, Never ("Off" is also accepted as a synonym for "Never") | +| CALICO_IPV4POOL_VXLAN | VXLAN Mode to use for the IPv4 Pool created at start up. If set to a value other than `Never`, `CALICO_IPV4POOL_IPIP` should not be set. [Default: `Never`] | Always, CrossSubnet, Never | +| CALICO_IPV4POOL_NAT_OUTGOING | Controls NAT Outgoing for the IPv4 Pool created at start up. [Default: `true`] | boolean | +| CALICO_IPV4POOL_NODE_SELECTOR | Controls the NodeSelector for the IPv4 Pool created at start up. [Default: `all()`] | [selector](../../resources/ippool.mdx#node-selector) | +| CALICO_IPV6POOL_CIDR | The IPv6 Pool to create if none exists at start up. It is invalid to define this variable and NO_DEFAULT_POOLS. [Default: ``] | IPv6 CIDR | +| CALICO_IPV6POOL_BLOCK_SIZE | Block size to use for the IPv6 POOL created at startup. Block size for IPv6 should be in the range 116-128 (inclusive) [Default: `122`] | int | +| CALICO_IPV6POOL_VXLAN | VXLAN Mode to use for the IPv6 Pool created at start up. [Default: `Never`] | Always, CrossSubnet, Never | +| CALICO_IPV6POOL_NAT_OUTGOING | Controls NAT Outgoing for the IPv6 Pool created at start up. [Default: `false`] | boolean | +| CALICO_IPV6POOL_NODE_SELECTOR | Controls the NodeSelector for the IPv6 Pool created at start up. [Default: `all()`] | [selector](../../resources/ippool.mdx#node-selector) | +| CALICO_IPV4POOL_DISABLE_BGP_EXPORT | Disable exporting routes over BGP for the IPv4 Pool created at start up. [Default: `false`] | boolean | +| CALICO_IPV6POOL_DISABLE_BGP_EXPORT | Disable exporting routes over BGP for the IPv6 Pool created at start up. [Default: `false`] | boolean | +| NO_DEFAULT_POOLS | Prevents $[prodname] from creating a default pool if one does not exist. [Default: `false`] | boolean | + +### Configuring BGP Networking + +BGP configuration for Calico nodes is normally configured through the [Node](../../resources/node.mdx), [BGPConfiguration](../../resources/bgpconfig.mdx), and [BGPPeer](../../resources/bgppeer.mdx) resources. +`$[nodecontainer]` also exposes some options to allow setting certain fields on these objects, as described +below. + +| Environment | Description | Schema | +| ------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------- | +| NODENAME | A unique identifier for this host. See [node name determination](#node-name-determination) for more details. | lowercase string | +| IP | The IPv4 address to assign this host or detection behavior at startup. Refer to [IP setting](#ip-setting) for the details of the behavior possible with this field. | IPv4 | +| IP6 | The IPv6 address to assign this host or detection behavior at startup. Refer to [IP setting](#ip-setting) for the details of the behavior possible with this field. | IPv6 | +| IP_AUTODETECTION_METHOD | The method to use to autodetect the IPv4 address for this host. This is only used when the IPv4 address is being autodetected. See [IP Autodetection methods](#ip-autodetection-methods) for details of the valid methods. [Default: `first-found`] | string | +| IP6_AUTODETECTION_METHOD | The method to use to autodetect the IPv6 address for this host. This is only used when the IPv6 address is being autodetected. See [IP Autodetection methods](#ip-autodetection-methods) for details of the valid methods. [Default: `first-found`] | string | +| AS | The AS number for this node. When specified, the value is saved in the node resource configuration for this host, overriding any previously configured value. When omitted, if an AS number has been previously configured in the node resource, that AS number is used for the peering. When omitted, if an AS number has not yet been configured in the node resource, the node will use the global value (see [example modifying Global BGP settings](../../../networking/configuring/bgp.mdx) for details.) | int | +| CALICO_ROUTER_ID | Sets the `router id` to use for BGP if no IPv4 address is set on the node. For an IPv6-only system, this may be set to `hash`. It then uses the hash of the nodename to create a 4 byte router id. See note below. [Default: ``] | string | +| CALICO_K8S_NODE_REF | The name of the corresponding node object in the Kubernetes API. When set, used for correlating this node with events from the Kubernetes API. | string | + +### Configuring Datastore Access + +| Environment | Description | Schema | +| -------------- | ------------------------------------------ | ------------------ | +| DATASTORE_TYPE | Type of datastore. [Default: `kubernetes`] | kubernetes, etcdv3 | + +#### Configuring Kubernetes Datastore Access + +| Environment | Description | Schema | +| ---------------- | ------------------------------------------------------------------------------ | ------ | +| KUBECONFIG | When using the Kubernetes datastore, the location of a kubeconfig file to use. | string | +| K8S_API_ENDPOINT | Location of the Kubernetes API. Not required if using kubeconfig. | string | +| K8S_CERT_FILE | Location of a client certificate for accessing the Kubernetes API. | string | +| K8S_KEY_FILE | Location of a client key for accessing the Kubernetes API. | string | +| K8S_CA_FILE | Location of a CA for accessing the Kubernetes API. | string | + +:::note + +When $[prodname] is configured to use the Kubernetes API as the datastore, the environments +used for BGP configuration are ignored—this includes selection of the node AS number (AS) +and all of the IP selection options (IP, IP6, IP_AUTODETECTION_METHOD, IP6_AUTODETECTION_METHOD). + +::: + +### Configuring Logging + +| Environment | Description | Schema | +| --------------------------- | -------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------- | +| CALICO_DISABLE_FILE_LOGGING | Disables logging to file. [Default: "false"] | string | +| CALICO_STARTUP_LOGLEVEL | The log severity above which startup `$[nodecontainer]` logs are sent to the stdout. [Default: `ERROR`] | DEBUG, INFO, WARNING, ERROR, CRITICAL, or NONE (case-insensitive) | + +### Configuring CNI Plugin + +`$[nodecontainer]` has a few options that are configurable based on the CNI plugin and CNI plugin +configuration used on the cluster. + +| Environment | Description | Schema | +| ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | +| USE_POD_CIDR | Use the Kubernetes `Node.Spec.PodCIDR` field when using host-local IPAM. Requires Kubernetes API datastore. This field is required when using the Kubernetes API datastore with host-local IPAM. [Default: false] | boolean | +| CALICO_MANAGE_CNI | Tells Calico to update the kubeconfig file at /host/etc/cni/net.d/calico-kubeconfig on credentials change. [Default: true] | boolean | + +### Other Environment Variables + +| Environment | Description | Schema | +| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------- | +| DISABLE_NODE_IP_CHECK | Skips checks for duplicate Node IPs. This can reduce the load on the cluster when a large number of Nodes are restarting. [Default: `false`] | boolean | +| WAIT_FOR_DATASTORE | Wait for connection to datastore before starting. If a successful connection is not made, node will shutdown. [Default: `false`] | boolean | +| CALICO_NETWORKING_BACKEND | The networking backend to use. In `bird` mode, Calico will provide BGP networking using the BIRD BGP daemon; VXLAN networking can also be used. In `vxlan` mode, only VXLAN networking is provided; BIRD and BGP are disabled. If set to `none` (also known as policy-only mode), both BIRD and VXLAN are disabled. [Default: `bird`] | bird, vxlan, none | +| CLUSTER_TYPE | Contains comma delimited list of indicators about this cluster. e.g. k8s, mesos, kubeadm, canal, bgp | string | + +## Appendix + +### Node name determination + +The `$[nodecontainer]` must know the name of the node on which it is running. The node name is used to +retrieve the [Node resource](../../resources/node.mdx) configured for this node if it exists, or to create a new node resource representing the node if it does not. It is +also used to associate the node with per-node [BGP configuration](../../resources/bgpconfig.mdx), [felix configuration](../../resources/felixconfig.mdx), and endpoints. + +When launched, the `$[nodecontainer]` container sets the node name according to the following order of precedence: + +1. The value specified in the `NODENAME` environment variable, if set. +1. The value specified in `/var/lib/calico/nodename`, if it exists. +1. The value specified in the `HOSTNAME` environment variable, if set. +1. The hostname as returned by the operating system, converted to lowercase. + +Once the node has determined its name, the value will be cached in `/var/lib/calico/nodename` for future use. + +For example, if given the following conditions: + +- `NODENAME=""` +- `/var/lib/calico/nodename` does not exist +- `HOSTNAME="host-A"` +- The operating system returns "host-A.internal.myorg.com" for the hostname + +$[nodecontainer] will use "host-a" for its name and will write the value in `/var/lib/calico/nodename`. If $[nodecontainer] +is then restarted, it will use the cached value of "host-a" read from the file on disk. + +### IP setting + +The IP (for IPv4) and IP6 (for IPv6) environment variables are used to set, +force autodetection, or disable auto detection of the address for the +appropriate IP version for the node. When the environment variable is set, +the address is saved in the +[node resource configuration](../../resources/node.mdx) +for this host, overriding any previously configured value. + +calico/node will attempt to detect subnet information from the host, and augment the provided address +if possible. + +#### IP setting special case values + +There are several special case values that can be set in the IP(6) environment variables, they are: + +- Not set or empty string: Any previously set address on the node + resource will be used. If no previous address is set on the node resource + the two versions behave differently: + - IP will do autodetection of the IPv4 address and set it on the node + resource. + - IP6 will not do autodetection. +- `autodetect`: Autodetection will always be performed for the IP address and + the detected address will overwrite any value configured in the node + resource. +- `none`: Autodetection will not be performed (this is useful to disable IPv4). + +### IP autodetection methods + +When $[prodname] is used for routing, each node must be configured with an IPv4 +address and/or an IPv6 address that will be used to route between +nodes. To eliminate node specific IP address configuration, the `$[nodecontainer]` +container can be configured to autodetect these IP addresses. In many systems, +there might be multiple physical interfaces on a host, or possibly multiple IP +addresses configured on a physical interface. In these cases, there are +multiple addresses to choose from and so autodetection of the correct address +can be tricky. + +The IP autodetection methods are provided to improve the selection of the +correct address, by limiting the selection based on suitable criteria for your +deployment. + +The following sections describe the available IP autodetection methods. + +#### first-found + +The `first-found` option enumerates all interface IP addresses and returns the +first valid IP address (based on IP version and type of address) on +the first valid interface. Certain known "local" interfaces +are omitted, such as the docker bridge. The order that both the interfaces +and the IP addresses are listed is system dependent. + +This is the default detection method. However, since this method only makes a +very simplified guess, it is recommended to either configure the node with a +specific IP address, or to use one of the other detection methods. + +e.g. + +``` +IP_AUTODETECTION_METHOD=first-found +IP6_AUTODETECTION_METHOD=first-found +``` + +#### kubernetes-internal-ip + +The `kubernetes-internal-ip` method will select the first internal IP address listed in the Kubernetes node's `Status.Addresses` field + +Example: + +``` +IP_AUTODETECTION_METHOD=kubernetes-internal-ip +IP6_AUTODETECTION_METHOD=kubernetes-internal-ip +``` + +#### can-reach=DESTINATION + +The `can-reach` method uses your local routing to determine which IP address +will be used to reach the supplied destination. Both IP addresses and domain +names may be used. + +Example using IP addresses: + +``` +IP_AUTODETECTION_METHOD=can-reach=8.8.8.8 +IP6_AUTODETECTION_METHOD=can-reach=2001:4860:4860::8888 +``` + +Example using domain names: + +``` +IP_AUTODETECTION_METHOD=can-reach=www.google.com +IP6_AUTODETECTION_METHOD=can-reach=www.google.com +``` + +#### interface=INTERFACE-REGEX + +The `interface` method uses the supplied interface [regular expression](https://pkg.go.dev/regexp) +to enumerate matching interfaces and to return the first IP address on +the first matching interface. The order that both the interfaces +and the IP addresses are listed is system dependent. + +Example with valid IP address on interface eth0, eth1, eth2 etc.: + +``` +IP_AUTODETECTION_METHOD=interface=eth.* +IP6_AUTODETECTION_METHOD=interface=eth.* +``` + +#### skip-interface=INTERFACE-REGEX + +The `skip-interface` method uses the supplied interface [regular expression](https://pkg.go.dev/regexp) +to exclude interfaces and to return the first IP address on the first +interface that does not match. The order that both the interfaces +and the IP addresses are listed is system dependent. + +Example with valid IP address on interface exclude enp6s0f0, eth0, eth1, eth2 etc.: + +``` +IP_AUTODETECTION_METHOD=skip-interface=enp6s0f0,eth.* +IP6_AUTODETECTION_METHOD=skip-interface=enp6s0f0,eth.* +``` + +#### cidr=CIDR + +The `cidr` method will select any IP address from the node that falls within the given CIDRs. For example: + +Example: + +``` +IP_AUTODETECTION_METHOD=cidr=10.0.1.0/24,10.0.2.0/24 +IP6_AUTODETECTION_METHOD=cidr=2001:4860::0/64 +``` + +### Node readiness + +The `calico/node` container supports an exec readiness endpoint. + +To access this endpoint, use the following command. + +```bash +docker exec calico-node /bin/calico-node [flag] +``` + +Substitute `[flag]` with one or more of the following. + +- `-bird-ready` +- `-bird6-ready` +- `-felix-ready` + +The BIRD readiness endpoint ensures that the BGP mesh is healthy by verifying that all BGP peers are established and +no graceful restart is in progress. + +### Setting `CALICO_ROUTER_ID` for IPv6 only system + +Setting CALICO_ROUTER_ID to value `hash` will use a hash of the configured nodename for the router ID. This should only be used in IPv6-only systems with no IPv4 address to use for the router ID. Since each node chooses its own router ID in isolation, it is possible for two nodes to pick the same ID resulting in a clash. The probability of such a clash grows with cluster size so this feature should not be used in a large cluster (500+ nodes). + + + + + +[installation]: /reference/installation/api.mdx \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/reference/component-resources/node/felix/configuration.mdx b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/node/felix/configuration.mdx new file mode 100644 index 0000000000..e960e1c808 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/node/felix/configuration.mdx @@ -0,0 +1,294 @@ +--- +description: Reference for Felix configuration parameters in Calico Cloud connected clusters covering environment variables, FelixConfiguration fields, and per-node overrides. +--- + +# Configuring Felix + +import FelixConfig from '../../../../_includes/components/FelixConfig'; + +:::note + +The following tables detail the configuration file and +environment variable parameters. For `FelixConfiguration` resource settings, +refer to [Felix Configuration Resource](../../../resources/felixconfig.mdx). + +::: + +Configuration for Felix is read from one of five possible locations, in order, as follows. + +1. Environment variables. +2. The Felix configuration file. +3. Host-specific `FelixConfiguration` resources (`node.`). +4. Selector-scoped `FelixConfiguration` resources (technology preview — see [Selector-scoped configuration](../../../resources/felixconfig.mdx#selector-scoped-configuration)). +5. The global `FelixConfiguration` resource (`default`). + +The value of any configuration parameter is the value read from the +_first_ location containing a value. For example, if an environment variable +contains a value, it takes top precedence. + +If not set in any of these locations, most configuration parameters have +defaults, and it should be rare to have to explicitly set them. + +The full list of parameters which can be set is as follows. + +## Spec + +### Datastore connection + + + +### Process: Feature detection/overrides + + + +### Process: Go runtime + + + +### Process: Health port and timeouts + + + +### Process: Logging + + + +### Process: Prometheus metrics + + + +### Dataplane: Common + + + +### Dataplane: iptables + + + +### Dataplane: nftables + + + +### Dataplane: eBPF + + + +### Dataplane: Windows + + + +### Dataplane: OpenStack support + + + +### Dataplane: XDP acceleration for iptables dataplane + + + +### Overlay: VXLAN overlay + + + +### Overlay: IP-in-IP + + + +### Overlay: Wireguard + + + +### Overlay: IPSec + + + +### Flow logs: Prometheus reports + + + +### Flow logs: Syslog reports + + + +### Flow logs: file reports + + + +### DNS logs / policy + + + +### L7 logs + + + +### AWS integration + + + +### Egress gateway + + + +### External network support + + + +### Packet capture + + + +### L7 proxy + + + +### Debug/test-only (generally unsupported) + + + +### Usage reporting + + + +## Environment variables + +The highest priority of configuration is that read from environment +variables. To set a configuration parameter via an environment variable, +set the environment variable formed by taking `FELIX_` and appending the +uppercase form of the variable name. For example, to set the etcd +address, set the environment variable `FELIX_ETCDADDR`. Other examples +include `FELIX_ETCDSCHEME`, `FELIX_ETCDKEYFILE`, `FELIX_ETCDCERTFILE`, +`FELIX_ETCDCAFILE`, `FELIX_FELIXHOSTNAME`, `FELIX_LOGFILEPATH` and +`FELIX_METADATAADDR`. + +## Configuration file + +On startup, Felix reads an ini-style configuration file. The path to +this file defaults to `/etc/calico/felix.cfg` but can be overridden +using the `-c` or `--config-file` options on the command line. If the +file exists, then it is read (ignoring section names) and all parameters +are set from it. + +In OpenStack, we recommend putting all configuration into configuration +files, since the etcd database is transient (and may be recreated by the +OpenStack plugin in certain error cases). However, in a Docker +environment the use of environment variables or etcd is often more +convenient. + +## Datastore + +Felix also reads configuration parameters from the datastore. It supports +a global setting and a per-host override. + +1. Get the current felixconfig settings. + + ```bash + kubectl get felixconfiguration.projectcalico.org default -o yaml --export > felix.yaml + ``` + +1. Modify logFilePath to your intended path, e.g. "/tmp/felix.log" + + ```bash + vim felix.yaml + ``` + + :::tip + + For a global change set name to "default". + For a node-specific change: set name to `node.`, e.g. "node.$[prodname]-node-1" + + ::: + +1. Replace the current felixconfig settings + + ```bash + kubectl replace -f felix.yaml + ``` + +For more information, see [Felix Configuration Resource](../../../resources/felixconfig.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/reference/component-resources/node/felix/index.mdx b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/node/felix/index.mdx new file mode 100644 index 0000000000..9b827cc62b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/node/felix/index.mdx @@ -0,0 +1,11 @@ +--- +description: Reference content for Felix in Calico Cloud connected clusters, the per-node daemon that programs routes and tiered policy rules into the Linux data plane. +hide_table_of_contents: true +--- + +# Felix + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/reference/component-resources/node/felix/prometheus.mdx b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/node/felix/prometheus.mdx new file mode 100644 index 0000000000..cf98b0a1a8 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/node/felix/prometheus.mdx @@ -0,0 +1,240 @@ +--- +description: Prometheus metrics reference for Felix in Calico Cloud connected clusters covering counters and gauges exposed for data plane health and policy evaluation. +--- + +# Monitoring Felix with Prometheus + +Felix can be configured to report a number of metrics through Prometheus. See the +[configuration reference](configuration.mdx) for how to enable metrics reporting. + +## Metric reference + +### Felix specific + +Felix exports a number of Prometheus metrics. The current set is as follows. Since some metrics +are tied to particular implementation choices inside Felix we can't make any hard guarantees that +metrics will persist across releases. However, we aim not to make any spurious changes to +existing metrics. + +| Name | Description | +| ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `felix_active_local_endpoints` | Number of active endpoints on this host. | +| `felix_active_local_policies` | Number of active policies on this host. | +| `felix_active_local_selectors` | Number of active selectors on this host. | +| `felix_active_local_tags` | Number of active tags on this host. | +| `felix_bpf_conntrack_cleaned` | Number of entries cleaned during a conntrack table sweep. | +| `felix_bpf_conntrack_cleaned_total` | Total number of entries cleaned during conntrack table sweeps, incremented for each clean individually. | +| `felix_bpf_conntrack_expired` | Number of entries cleaned during a conntrack table sweep due to expiration. | +| `felix_bpf_conntrack_expired_total` | Total number of entries cleaned during conntrack table sweep due to expiration - by reason. | +| `felix_bpf_conntrack_inforeader_blocks` | Conntrack InfoReader would-blocks. | +| `felix_bpf_conntrack_stale_nat` | Number of entries cleaned during a conntrack table sweep due to stale NAT. | +| `felix_bpf_conntrack_stale_nat_total` | Total number of entries cleaned during conntrack table sweeps due to stale NAT. | +| `felix_bpf_conntrack_sweeps` | Number of conntrack table sweeps made so far. | +| `felix_bpf_conntrack_used` | Number of used entries visited during a conntrack table sweep. | +| `felix_bpf_conntrack_sweep_duration` | Conntrack sweep execution time (ns). | +| `felix_bpf_num_ip_sets` | Number of BPF IP sets managed in the data plane. | +| `felix_calc_graph_output_events` | Number of events emitted by the calculation graph. | +| `felix_calc_graph_update_time_seconds` | Seconds to update calculation graph for each datastore OnUpdate call. | +| `felix_calc_graph_updates_processed` | Number of datastore updates processed by the calculation graph. | +| `felix_cluster_num_host_endpoints` | Total number of host endpoints cluster-wide. | +| `felix_cluster_num_hosts` | Total number of $[prodname] hosts in the cluster. | +| `felix_cluster_num_policies` | Total number of policies in the cluster. | +| `felix_cluster_num_profiles` | Total number of profiles in the cluster. | +| `felix_cluster_num_tiers` | Total number of $[prodname] tiers in the cluster. | +| `felix_cluster_num_workload_endpoints` | Total number of workload endpoints cluster-wide. | +| `felix_egress_gateway_remote_polls{status="total"}` | Total number of remote egress gateway pods that Felix is polling for health/connectivity. Only egress gateways with a named "health" port will be polled. | +| `felix_egress_gateway_remote_polls{status="up"}` | Total number of remote egress gateway pods that have successful probes. | +| `felix_egress_gateway_remote_polls{status="probe-failed"}` | Total number of remote egress gateway pods that have failed probes. | +| `felix_exec_time_micros` | Summary of time taken to fork/exec child processes | +| `felix_int_dataplane_addr_msg_batch_size` | Number of interface address messages processed in each batch. Higher values indicate we're doing more batching to try to keep up. | +| `felix_int_dataplane_apply_time_seconds` | Time in seconds that it took to apply a data plane update. | +| `felix_int_dataplane_failures` | Number of times data plane updates failed and will be retried. | +| `felix_int_dataplane_iface_msg_batch_size` | Number of interface state messages processed in each batch. Higher values indicate we're doing more batching to try to keep up. | +| `felix_int_dataplane_messages` | Number data plane messages by type. | +| `felix_int_dataplane_msg_batch_size` | Number of messages processed in each batch. Higher values indicate we're doing more batching to try to keep up. | +| `felix_ipsec_bindings_total` | Total number of IPsec bindings. | +| `felix_ipsec_errors` | Number of IPsec command failures. | +| `felix_ipset_calls` | Number of ipset commands executed. | +| `felix_ipset_errors` | Number of ipset command failures. | +| `felix_ipset_lines_executed` | Number of ipset operations executed. | +| `felix_ipsets_calico` | Number of active $[prodname] IP sets. | +| `felix_ipsets_total` | Total number of active IP sets. | +| `felix_iptables_chains` | Number of active iptables chains. | +| `felix_iptables_lines_executed` | Number of iptables rule updates executed. | +| `felix_iptables_lock_acquire_secs` | Time taken to acquire the iptables lock. | +| `felix_iptables_lock_retries` | Number of times the iptables lock was already held and felix had to retry to acquire it. | +| `felix_iptables_restore_calls` | Number of iptables-restore calls. | +| `felix_iptables_restore_errors` | Number of iptables-restore errors. | +| `felix_iptables_rules` | Number of active iptables rules. | +| `felix_iptables_save_calls` | Number of iptables-save calls. | +| `felix_iptables_save_errors` | Number of iptables-save errors. | +| `felix_log_errors` | Number of errors encountered while logging. | +| `felix_logs_dropped` | Number of logs dropped because the output stream was blocked. | +| `felix_reporter_log_errors` | Number of errors encountered while logging in the Syslog. | +| `felix_reporter_logs_dropped` | Number of logs dropped because the output was blocked in the Syslog reporter. | +| `felix_resync_state` | Current datastore state. | +| `felix_resyncs_started` | Number of times Felix has started resyncing with the datastore. | +| `felix_route_table_list_seconds` | Time taken to list all the interfaces during a resync. | +| `felix_route_table_per_iface_sync_seconds` | Time taken to sync each interface | + +Prometheus metrics are self-documenting, with metrics turned on, `curl` can be used to list the +metrics along with their help text and type information. + +```bash +curl -s http://localhost:9091/metrics | head +``` + +Example response: + +``` +# HELP felix_active_local_endpoints Number of active endpoints on this host. +# TYPE felix_active_local_endpoints gauge +felix_active_local_endpoints 91 +# HELP felix_active_local_policies Number of active policies on this host. +# TYPE felix_active_local_policies gauge +felix_active_local_policies 0 +# HELP felix_active_local_selectors Number of active selectors on this host. +# TYPE felix_active_local_selectors gauge +felix_active_local_selectors 82 +... +``` + +### Label indexing metrics + +The label index is a subcomponent of Felix that is responsible for calculating the set of endpoints and network sets +that match each selector that is in an active policy rule. Policy rules are active on a particular node if the policy +they belong to selects a workload or host endpoint on that node with its top-level selector (in `spec.selector`). +Inactive policies have minimal CPU cost because their selectors do not get indexed. + +Since the label index must match the active selectors against _all_ endpoints and network sets in the cluster, its +performance is critical and it supports various optimizations to minimize CPU usage. Its metrics can be used to +check that the optimizations are active for your policy set. + +#### `felix_label_index_num_endpoints` + +Reports the total number of endpoints (and similar objects such as network sets) being tracked by the index. +This should match the number of endpoints and network sets in your cluster. + +#### `felix_label_index_num_active_selectors{optimized="true|false"}` + +Reports the total number of active selectors, broken into `optimized="true"` and `optimized="false"` sub-totals. + +The `optimized="true"` total tracks the number of selectors that the label index was able to optimize. Those +selectors should be calculated efficiently even in clusters with hundreds of thousands of endpoints. In general the +CPU used to calculate them should be proportional to the number of endpoints that match them and the churn rate of +_those_ endpoints. + +The `optimized="false"` total tracks the number of selectors that could not be optimized. Unoptimized selectors are +much more costly to calculate; the CPU used to calculate them is proportional to the number of endpoints +in the cluster and their churn rate. It is generally OK to have a handful of unoptimized selectors, +but if many selectors are unoptimized the CPU usage can be substantial at high scale. + +For more information on writing selectors that can be optimized, see the [this](../../../resources/networkpolicy.mdx#selector-performance-in-entityrules) +section of the `NetworkPolicy` reference. + +#### `felix_label_index_selector_evals{result="true|false"}` + +Counts the total number of times that a selector was evaluated vs an endpoint to determine if it matches, broken +down by match (`true`) or no-match (`false`). The ratio of match to no-match shows how effective the selector +indexing optimizations are for your policy set. The more effectively the label index can optimize the selectors, +the fewer "no-match" results it will report relative to "match". + +If you have more than a handful of active selectors and `felix_label_index_selector_evals{result="false"}` is many +times `felix_label_index_selector_evals{result="true"}` then it is likely that some selectors in the policy set are +not being optimized effectively. + +#### `felix_label_index_strategy_evals{strategy="..."}` + +This is a technical statistic that shows how many times the label index has employed each optimization +strategy that it has available. The strategies will likely evolve over time but, at time of writing, they are +as follows: + +- `endpoint-full-scan`: the least efficient fall back strategy for unoptimized selectors. The index + scanned _all_ endpoints to find the matches for a selector. + +- `endpoint|parent-no-match`: the most efficient strategy; the index was able to prove that nothing matched the + selector so it was able to skip the scan entirely. + +- `endpoint|parent-single-value`: the label index was able to limit the scan to only those endpoints/parents that + have a particular label and value combination. For example, selector `label == "value"` would only scan items that + had exactly that label set to "value". + +- `endpoint|parent-multi-value`: the label index was able to limit the scan to only those endpoints/parents that + have a particular label and one of a few values. For example, selector `label in {"a", "b")` would only scan items + that had exactly that label with one of the given values. + +- `endpoint|parent-label-name`: the label index was able to limit the scan to only those endpoints/parents that + have a particular label (but was unable to limit it to a particular subset of values). For example, `has(label)` + would result in that kind of scan. + +Terminology: here "endpoint" means "endpoint or NetworkSet" and "parent" is Felix's internal name for resources like +Kubernetes Namespaces. A "parent" scan means that the label index scanned all endpoints that have a parent +matching the strategy. + +### CPU / memory metrics + +Felix also exports the default set of metrics that Prometheus makes available. Currently, those +include: + +| Name | Description | +| ---------------------------------- | ------------------------------------------------------------------------------------------- | +| `go_gc_duration_seconds` | A summary of the GC invocation durations. | +| `go_goroutines` | Number of goroutines that currently exist. | +| `go_info` | Go version. | +| `go_memstats_alloc_bytes` | Number of bytes allocated and still in use. | +| `go_memstats_alloc_bytes_total` | Total number of bytes allocated, even if freed. | +| `go_memstats_buck_hash_sys_bytes` | Number of bytes used by the profiling bucket hash table. | +| `go_memstats_frees_total` | Total number of frees. | +| `go_memstats_gc_cpu_fraction` | The fraction of this program’s available CPU time used by the GC since the program started. | +| `go_memstats_gc_sys_bytes` | Number of bytes used for garbage collection system metadata. | +| `go_memstats_heap_alloc_bytes` | Number of heap bytes allocated and still in use. | +| `go_memstats_heap_idle_bytes` | Number of heap bytes waiting to be used. | +| `go_memstats_heap_inuse_bytes` | Number of heap bytes that are in use. | +| `go_memstats_heap_objects` | Number of allocated objects. | +| `go_memstats_heap_released_bytes` | Number of heap bytes released to OS. | +| `go_memstats_heap_sys_bytes` | Number of heap bytes obtained from system. | +| `go_memstats_last_gc_time_seconds` | Number of seconds since 1970 of last garbage collection. | +| `go_memstats_lookups_total` | Total number of pointer lookups. | +| `go_memstats_mallocs_total` | Total number of mallocs. | +| `go_memstats_mcache_inuse_bytes` | Number of bytes in use by mcache structures. | +| `go_memstats_mcache_sys_bytes` | Number of bytes used for mcache structures obtained from system. | +| `go_memstats_mspan_inuse_bytes` | Number of bytes in use by mspan structures. | +| `go_memstats_mspan_sys_bytes` | Number of bytes used for mspan structures obtained from system. | +| `go_memstats_next_gc_bytes` | Number of heap bytes when next garbage collection will take place. | +| `go_memstats_other_sys_bytes` | Number of bytes used for other system allocations. | +| `go_memstats_stack_inuse_bytes` | Number of bytes in use by the stack allocator. | +| `go_memstats_stack_sys_bytes` | Number of bytes obtained from system for stack allocator. | +| `go_memstats_sys_bytes` | Number of bytes obtained by system. Sum of all system allocations. | +| `go_threads` | Number of OS threads created. | +| `process_cpu_seconds_total` | Total user and system CPU time spent in seconds. | +| `process_max_fds` | Maximum number of open file descriptors. | +| `process_open_fds` | Number of open file descriptors. | +| `process_resident_memory_bytes` | Resident memory size in bytes. | +| `process_start_time_seconds` | Start time of the process since Unix epoch in seconds. | +| `process_virtual_memory_bytes` | Virtual memory size in bytes. | +| `process_virtual_memory_max_bytes` | Maximum amount of virtual memory available in bytes. | + +### WireGuard Metrics + +Felix also exports WireGuard device stats if found/detected. Can be disabled via Felix configuration. + +| Name | Description | +| ------------------------------------ | ------------------------------------------------------------------------------------------------- | +| `wireguard_meta` | Gauge. Device / interface information for a felix/calico node, values are in this metric's labels | +| `wireguard_bytes_rcvd` | Counter. Current bytes received from a peer identified by a peer public key and endpoint | +| `wireguard_bytes_sent` | Counter. Current bytes sent to a peer identified by a peer public key and endpoint | +| `wireguard_latest_handshake_seconds` | Gauge. Last handshake with a peer, Unix timestamp in seconds. | + + +### Load Balancer Metrics + +Felix exports statistics for monitoring load balancing. + +#### `felix_bpf_conntrack_maglev_entries_total{destination="local|remote",ip_family="4|6"}` + +Total number of Maglev entries in conntrack table broken down by IP version, and, whether the destination backend +is remote (we're acting as a frontend) or local (we're the backend node). + +#### `felix_bpf_lb_maintenance` + +Can be 1 or 0 (true or false). Indicates whether Felix's node has been marked for maintenance. \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/reference/component-resources/node/index.mdx b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/node/index.mdx new file mode 100644 index 0000000000..d9017ea030 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/component-resources/node/index.mdx @@ -0,0 +1,11 @@ +--- +description: Reference content for the cnx-node component in Calico Cloud connected clusters covering Felix, BIRD, and the per-node agent configuration. +hide_table_of_contents: true +--- + +# Calico Cloud node (cnx-node) + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/reference/faq.mdx b/calico-cloud_versioned_docs/version-24-1/reference/faq.mdx new file mode 100644 index 0000000000..2e25a569df --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/faq.mdx @@ -0,0 +1,442 @@ +--- +description: Frequently asked questions about Calico Cloud covering connected clusters, the web console, observability, and platform support. +--- + +# Frequently asked questions + +## Why use $[prodname]? + +The problem $[prodname] tries to solve is the networking of workloads (VMs, +containers, etc) in a high scale environment. Existing L2-based methods +for solving this problem have problems at high scale. Compared to these, +we think $[prodname] is more scalable, simpler, and more flexible. We think +you should look into it if you have more than a handful of nodes on a +single site. + +$[prodname] also provides a rich network security model that +allows operators and developers to declare intent-based network security +policy that is automatically rendered into distributed firewall rules +across a cluster of containers, VMs, and/or servers. + +For a more detailed discussion of this topic, see our blog post at +[Why Calico?](https://www.projectcalico.org/why-calico/). + +## Does $[prodname] work with IPv6? + +Yes! $[prodname]'s core components support IPv6 out of the box. However, +not all orchestrators that we integrate with support IPv6 yet. + +## Why does my container have a route to 169.254.1.1? + +In a $[prodname] network, each host acts as a gateway router for the +workloads that it hosts. In container deployments, $[prodname] uses +169.254.1.1 as the address for the $[prodname] router. By using a +link-local address, $[prodname] saves precious IP addresses and avoids +burdening the user with configuring a suitable address. + +While the routing table may look a little odd to someone who is used to +configuring LAN networking, using explicit routes rather than +subnet-local gateways is fairly common in WAN networking. + +## Why isn't $[prodname] working with a containerized kubelet? + +$[prodname] hosted install places the necessary CNI binaries and config on each +Kubernetes node in a directory on the host as specified in the manifest. By +default it places binaries in /opt/cni/bin and config /etc/cni/net.d. + +When running the kubelet as a container using hyperkube, +you need to make sure that the containerized kubelet can see the CNI network +plugins and config that have been installed by mounting them into the kubelet container. + +For example add the following arguments to the kubelet-wrapper service: + +``` +--volume /etc/cni/net.d:/etc/cni/net.d \ +--volume /opt/cni/bin:/opt/cni/bin \ +``` + +Without the above volume mounts, the kubelet will not call the $[prodname] CNI binaries, and so +$[prodname] [workload endpoints](resources/workloadendpoint.mdx) will +not be created, and $[prodname] policy will not be enforced. + +## How do I view $[prodname] CNI logs? + +The $[prodname] CNI plugin emits logs to stderr, which are then logged out by the kubelet. Where these logs end up +depend on how your kubelet is configured. For deployments using `systemd`, you can do this via `journalctl`. + +The log level can be configured via the CNI network configuration file, by changing the value of the +key `log_level`. See [Configuring the $[prodname] CNI plugins](component-resources/configuration.mdx) for more information. + +CNI plugin logs can also be found in `/var/log/calico/cni`. + +## How do I configure the pod IP range? + +When using $[prodname] IPAM, IP addresses are assigned from [IP Pools](resources/ippool.mdx). + +By default, all enabled IP pools are used. However, you can specify which IP pools to use for IP address management in the [CNI network config](component-resources/configuration.mdx#ipam), +or on a per-pod basis using [Kubernetes annotations](component-resources/configuration.mdx#using-kubernetes-annotations). + +## How do I assign a specific IP address to a pod? + +For most use cases it's not necessary to assign specific IP addresses to a Kubernetes pod and it's recommended to use Kubernetes services instead. +However, if you do need to assign a particular address to a pod, $[prodname] provides two ways of doing this: + +- You can request an IP that is available in $[prodname] IPAM using the `cni.projectcalico.org/ipAddrs` annotation. +- You can request an IP using the `cni.projectcalico.org/ipAddrsNoIpam` annotation. Note that this annotation bypasses the configured IPAM plugin, and thus in most cases it is recommended to use the above annotation. + +See the [Requesting a specific IP address](component-resources/configuration.mdx#requesting-a-specific-ip-address) section in the CNI plugin reference documentation for more details. + +## Why can't I see the 169.254.1.1 address mentioned above on my host? + +$[prodname] tries hard to avoid interfering with any other configuration +on the host. Rather than adding the gateway address to the host side +of each workload interface, $[prodname] sets the `proxy_arp` flag on the +interface. This makes the host behave like a gateway, responding to +ARPs for 169.254.1.1 without having to actually allocate the IP address +to the interface. + +## Why do all cali\* interfaces have the MAC address ee:ee:ee:ee:ee:ee:? + +In some setups the kernel is unable to generate a persistent MAC address and so +$[prodname] assigns a MAC address itself. Since $[prodname] uses +point-to-point routed interfaces, traffic does not reach the data link layer +so the MAC Address is never used and can therefore be the same for all the +cali\* interfaces. + +## Can I prevent my Kubernetes pods from initiating outgoing connections? + +Yes! The Kubernetes [`NetworkPolicy`](https://kubernetes.io/docs/concepts/services-networking/network-policies/) +API added support for egress policies in v1.8. You can also use `calicoctl` +to configure egress policy to prevent Kubernetes pods from initiating outgoing +connections based on the full set of supported $[prodname] policy primitives +including labels, Kubernetes namespaces, CIDRs, and ports. + +## I've heard $[prodname] uses proxy ARP, doesn't proxy ARP cause a lot of problems? + +It can, but not in the way that $[prodname] uses it. + +In container deployments, $[prodname] only uses proxy ARP for resolving the +169.254.1.1 address. The routing table inside the container ensures +that all traffic goes via the 169.254.1.1 gateway so that is the only +IP that will be ARPed by the container. + +## Is $[prodname] compliant with PCI/DSS requirements? + +PCI certification applies to the whole end-to-end system, of which +$[prodname] would be a part. We understand that most current solutions use +VLANs, but after studying the PCI requirements documents, we believe +that $[prodname] does meet those requirements and that nothing in the +documents _mandates_ the use of VLANs. + +## How do I enable IP-in-IP and NAT outgoing on an IP pool? + +1. Retrieve current IP pool config. + + ```bash + calicoctl get ipPool --export -o yaml > pool.yaml + ``` + +2. Modify IP pool config. + + Modify the pool's spec to enable IP-in-IP and NAT outgoing. (See + [IP pools](resources/ippool.mdx) + for other settings that can be edited.) + + ```shell + - apiVersion: projectcalico.org/v3 + kind: IPPool + metadata: + name: ippool-1 + spec: + cidr: 192.168.0.0/16 + ipipMode: Always + natOutgoing: true + ``` + +3. Load the modified file. + + ```bash + kubectl replace -f pool.yaml + ``` + +## How does $[prodname] maintain saved state? + +State is saved in a few places in a $[prodname] deployment, depending on +whether it's global or local state. + +Local state is state that belongs on a single compute host, associated +with a single running Felix instance (things like kernel routes, tap +devices etc.). Local state is entirely stored by the Linux kernel on the +host, with Felix storing it only as a temporary mirror. This makes Felix +effectively stateless, with the kernel acting as a backing data store on +one side and Kubernetes (kdd) as a data source on the other. + +If Felix is restarted, it learns current local state by interrogating +the kernel at start up. It then reads from the etcd datastore all the local state +which it should have, and updates the kernel to match. This approach has +strong resiliency benefits, in that if Felix restarts you don't suddenly +lose access to your VMs or containers. As long as the Linux kernel is +running, you've still got full functionality. + +The bulk of global state is mastered in whatever component hosts the +plugin. + +- In certain cases, `etcd` itself contains the master copy of + the data. This is because some Docker deployments have an `etcd` + cluster that has the required resiliency characteristics, used to + store all system configuration and so `etcd` is configured so as to + be a suitable store for critical data. +- In other orchestration systems, it may be stored in distributed + databases, either owned directly by the plugin or by the + orchestrator itself. + +The only other state storage in a $[prodname] network is in the BGP sessions, +which approximate a distributed database of routes. This BGP state is +simply a replicated copy of the per-host routes configured by Felix +based on the global state provided by the orchestrator. + +This makes the $[prodname] design very simple, because we store very little +state. All of our components can be shut down and restarted without risk, +because they resynchronize state as necessary. This makes modeling +their behavior extremely simple, reducing the complexity of bugs. + +## I heard $[prodname] is suggesting layer 2: I thought you were layer 3! What's happening? + +It's important to distinguish what $[prodname] provides to the workloads +hosted in a data center (a purely layer 3 network) with what the $[prodname] +project _recommends_ operators use to build their underlying network +fabric. + +$[prodname]'s core principle is that _applications_ and _workloads_ +overwhelmingly need only IP connectivity to communicate. For this reason +we build an IP-forwarded network to connect the tenant applications and +workloads to each other and the broader world. + +However, the underlying physical fabric obviously needs to be set up +too. Here, $[prodname] has discussed how both a layer 2 (see +[here](architecture/design/l2-interconnect-fabric.mdx)) +or a layer 3 (see +[here](architecture/design/l3-interconnect-fabric.mdx)) +fabric +could be integrated with $[prodname]. This is one of the great strengths of +the $[prodname] model: it allows the infrastructure to be decoupled from what +we show to the tenant applications and workloads. + +We have some thoughts on different interconnect approaches (as noted +above), but just because we say that there are layer 2 and layer 3 ways +of building the fabric, and that those decisions may have an impact on +route scale, does not mean that $[prodname] is "going back to Ethernet" or +that we're recommending layer 2 for tenant applications. In all cases we +forward on IP packets, no matter what architecture is used to build the +fabric. + +## How do I control policy/connectivity without virtual/physical firewalls? + +$[prodname] provides an extremely rich security policy model, applying policy at the first and last hop +of the routed traffic within the $[prodname] network (the source and +destination compute hosts). + +This model is substantially more robust to failure than a centralized +firewall-based model. In particular, the $[prodname] approach has no +single point of failure: if the device enforcing the firewall has failed +then so has one of the workloads involved in the traffic (because the +firewall is enforced by the compute host). + +This model is also extremely amenable to scaling out. Because we have a +central repository of policy configuration, but apply it at the edges of +the network (the hosts) where it is needed, we automatically ensure that +the rules match the topology of the data center. This allows easy +scaling out, and gives us all the advantages of a single firewall (one +place to manage the rules), but none of the disadvantages (single points +of failure, state sharing, hairpinning of traffic, etc.). + +Lastly, we decouple the reachability of nodes and the policy applied to +them. We use BGP to distribute the topology of the network, telling +every node how to get to every endpoint in case two endpoints need to +communicate. We use policy to decide _if_ those two nodes should +communicate, and if so, how. If policy changes and two endpoints should +now communicate, where before they shouldn’t have, all we have to do is +update policy: the reachability information does not change. If later +they should be denied the ability to communicate, the policy is updated +again, and again the reachability doesn’t have to change. + +## Why isn't the `-p` flag on `docker run` working as expected? + +The `-p` flag tells Docker to set up port mapping to connect a port on the +Docker host to a port on your container via the `docker0` bridge. + +If a host's containers are connected to the `docker0` bridge interface, $[prodname] +would be unable to enforce security rules between workloads on the same host; +all containers on the bridge would be able to communicate with one other. + +You can securely configure port mapping by following [Configure outgoing NAT](../networking/configuring/workloads-outside-cluster.mdx. + +## Can $[prodname] containers use any IP address within a pool, even subnet network/broadcast addresses? + +Yes! $[prodname] is fully routed, so all IP address within a $[prodname] pool are usable as +private IP addresses to assign to a workload. This means addresses commonly +reserved in a L2 subnet, such as IPv4 addresses ending in .0 or .255, are perfectly +okay to use. + +## How do I get network traffic into and out of my $[prodname] cluster? + +The recommended way to get traffic to/from your $[prodname] network is by peering to +your existing data center L3 routers using BGP and by assigning globally +routable IPs (public IPs) to containers that need to be accessed from the internet. +This allows incoming traffic to be routed directly to your containers without the +need for NAT. This flat L3 approach delivers exceptional network scalability +and performance. + +A common scenario is for your container hosts to be on their own +isolated layer 2 network, like a rack in your server room or an entire data +center. Access to that network is via a router, which also is the default +router for all the container hosts. + +If this describes your infrastructure, +[Configure outgoing NAT](../networking/configuring/workloads-outside-cluster.mdx explains in more detail +what to do. Otherwise, if you have a layer 3 (IP) fabric, then there are +detailed datacenter networking recommendations given +in [$[prodname] over IP fabrics](architecture/design/l3-interconnect-fabric.mdx). +We'd also encourage you to [get in touch](https://www.projectcalico.org/contact) +to discuss your environment. + +### How can I enable NAT for outgoing traffic from containers with private IP addresses? + +If you want to allow containers with private IP addresses to be able to access the +internet then you can use your data center's existing outbound NAT capabilities +(typically provided by the data center's border routers). + +Alternatively you can use $[prodname]'s built in outbound NAT capability by enabling it on any +$[prodname] IP pool. In this case $[prodname] will perform outbound NAT locally on the compute +node on which each container is hosted. + +```bash +cat < + natOutgoing: true +EOF +``` + +Where `` is the CIDR of your IP pool, for example `192.168.0.0/16`. + +Remember: the security profile for the container will need to allow traffic to the +internet as well. Refer to the appropriate guide for your orchestration +system for details on how to configure policy. + +### How can I enable NAT for incoming traffic to containers with private IP addresses? + +As discussed, the recommended way to get traffic to containers that +need to be accessed from the internet is to give them public IP addresses and +to configure $[prodname] to peer with the data center's existing L3 routers. + +In cases where this is not possible then you can configure incoming NAT +(also known as DNAT) on your data centers existing border routers. Alternatively +you can configure incoming NAT with port mapping on the host on which the container +is running on. + +1. Create a new chain called `expose-ports` to hold the NAT rules. + + ```bash + iptables -t nat -N expose-ports + ``` + +1. Jump to that chain from the `OUTPUT` and `PREROUTING` chains. + + ```bash + iptables -t nat -A OUTPUT -j expose-ports + iptables -t nat -A PREROUTING -j expose-ports + ``` + + :::tip + + The `OUTPUT` chain is hit by traffic originating on the host itself; + the `PREROUTING` chain is hit by traffic coming from elsewhere. + + ::: + +1. For each port you want to expose, add a rule to the + expose-ports chain, replacing `` with the host IP that you + want to use to expose the port and `` with the host port. + + ```bash + iptables -t nat -A expose-ports -p tcp --destination \ + --dport -j DNAT --to : + ``` + +For example, you have a container to which you've assigned the `CALICO_IP` +of 192.168.7.4, and you have NGINX running on port 8080 inside the container. +If you want to expose this service on port 80 and your host has IP 192.0.2.1, +then you could run the following commands: + +```bash +iptables -t nat -N expose-ports +iptables -t nat -A OUTPUT -j expose-ports +iptables -t nat -A PREROUTING -j expose-ports + +iptables -t nat -A expose-ports -p tcp --destination 192.0.2.1 --dport 80 -j DNAT --to 192.168.7.4:8080 +``` + +The commands will need to be run each time the host is restarted. + +Remember: the security profile for the container will need to allow traffic to the exposed port as well. +Refer to the appropriate guide for your orchestration system for details on how to configure policy. + +### Can I run $[prodname] in a public cloud environment? + +Yes. If you are running in a public cloud that doesn't allow either L3 peering or L2 connectivity between $[prodname] hosts then you can enable IP-in-IP in your $[prodname] IP pool: + +```bash +cat < + ipipMode: Always + natOutgoing: true +EOF +``` + +$[prodname] will then route traffic between $[prodname] hosts using IP-in-IP. + +For best performance in AWS, you can disable [Source/Destination Check](resources/felixconfig.mdx#spec) instead of using IP-in-IP or VXLAN; but only if all your instances are in the same subnet of your VPC. The setting must be `Disable` for the EC2 instance(s) to process traffic not matching the host interface IP address. This is also applicable if your cluster is spread across multiple subnets. If your cluster traffic crosses subnets, set `ipipMode` (or `vxlanMode`) to `CrossSubnet` to reduce the encapsulation overhead. Check [configuring overlay networking](../networking/configuring/vxlan-ipip.mdx) for the details. + +You can disable Source/Destination Check using [Felix configuration](resources/felixconfig.mdx), the AWS CLI, or the EC2 console. For example, using the AWS CLI: + +```bash +aws ec2 modify-instance-attribute --instance-id --source-dest-check "{\"Value\": false}" + +cat < + natOutgoing: true +EOF +``` + +### On AWS with IP-in-IP, why do I see no connectivity between workloads or only see connectivity if I ping in both directions? + +By default, AWS security groups block incoming IP-in-IP traffic. + +However, if an instance has recently sent some IP-in-IP traffic out when it receives some incoming IP-in-IP traffic, +then AWS sees that as a response to an outgoing connection and it allows the incoming traffic. This leads to some very +confusing behavior where traffic can be blocked and then suddenly start working! + +To resolve the issue, add a rule to your security groups that allows inbound and outbound IP-in-IP traffic (IP protocol +number 4) between your hosts. + +## Can Calico do IP multicast? + +Calico is a routed L3 network where each pod gets a /32. There's no broadcast domain for pods. +That means that multicast doesn't just work as a side effect of broadcast. To get multicast to +work, the host needs to act as a multicast gateway of some kind. Calico's architecture was designed +to extend to cover that case but it's not part of the product as yet. diff --git a/calico-cloud_versioned_docs/version-24-1/reference/feature-lifecycle-reference.mdx b/calico-cloud_versioned_docs/version-24-1/reference/feature-lifecycle-reference.mdx new file mode 100644 index 0000000000..8fb6e53813 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/feature-lifecycle-reference.mdx @@ -0,0 +1,23 @@ +--- +description: See which Calico Cloud features are currently in technology preview or deprecated, and the Calico Enterprise release each change tracks. +title: Feature lifecycle reference +sidebar_label: Feature lifecycle reference +--- + +import { TechPreviewTable, DeprecatedFeaturesTable } from '@site/src/components/FeatureStatusTable'; + +# Feature lifecycle reference + +This page provides the status of features that have recently been released in technology preview or that have been deprecated. + +## Technology preview features + +Technology preview features are for evaluation and feedback only and are **not supported for production use**. Behavior, APIs, and configuration may change before a feature reaches general availability. + + + +## Deprecated and removed features + +Deprecated features are still present and supported, but they are scheduled for removal. You should consider migrating away from a deprecated feature before it is removed. + + diff --git a/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/connectivity.mdx b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/connectivity.mdx new file mode 100644 index 0000000000..6dfce97390 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/connectivity.mdx @@ -0,0 +1,91 @@ +--- +description: Reference for the Calico Cloud failsafe policy that protects host endpoints in connected clusters from being cut off by misconfigured host policy. +--- + +# Creating policy for basic connectivity + +When a host endpoint is added, if there is no security policy for that +endpoint, $[prodname] will default to denying traffic to/from that endpoint, +except for traffic that is allowed by the [failsafe rules](failsafe.mdx). + +While the [failsafe rules](failsafe.mdx) provide protection against removing all +connectivity to a host: + +- They are overly broad in allowing inbound SSH on any interface and + allowing traffic out to etcd's ports on any interface. + +- Depending on your network, they may not cover all the ports that are + required; for example, your network may rely on allowing ICMP, + or DHCP. + +Therefore, we recommend creating a failsafe $[prodname] security policy that +is tailored to your environment. The example command below shows one +example of how you might do that; the command uses `calicoctl` to create a single +policy resource, which: + +- Applies to all known endpoints. +- Allows inbound ssh access from a defined “management” subnet. +- Allows outbound connectivity to etcd on a particular IP; if you have multiple etcd servers you should duplicate the rule for each destination. +- Allows inbound ICMP. +- Allows outbound UDP on port 67, for DHCP. + +When running this command, replace the placeholders in angle brackets with +appropriate values for your deployment. + +{/* */} + +```bash +cat <" + destination: + ports: [22] + - action: Allow + protocol: ICMP + egress: + - action: Allow + protocol: TCP + destination: + nets: [/32] + ports: [] + - action: Allow + protocol: TCP + destination: + nets: [] + - action: Allow + protocol: UDP + destination: + ports: [67] +EOF +``` + +Once you have such a policy in place, you may want to disable the +[failsafe rules](failsafe.mdx). + +:::note + +Packets that reach the end of the list of rules fall-through to the +next policy (sorted by the `order` field). +The selector in the policy, `all()`, will match _all_ endpoints, +including any workload endpoints. If you have workload endpoints as +well as host endpoints then you may wish to use a more restrictive +selector. For example, you could label management interfaces with +label `endpoint_type = management` and then use selector +`endpoint_type == "management"` +If you are using $[prodname] for networking workloads, you should add +inbound and outbound rules to allow BGP: add an ingress and egress rule +to allow TCP traffic to destination port 179. + +::: diff --git a/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/conntrack.mdx b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/conntrack.mdx new file mode 100644 index 0000000000..45db7470c6 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/conntrack.mdx @@ -0,0 +1,29 @@ +--- +description: Reference covering Linux conntrack workarounds for Calico Cloud host endpoint policy when stateful tracking interferes with packet flow in connected clusters. +--- + +# Connection tracking + +$[prodname] uses Linux's connection tracking ('conntrack') as an important +optimization to its processing. It generally means that $[prodname] only needs to +check its policies for the first packet in an allowed flow—between a pair of +IP addresses and ports—and then conntrack automatically allows further +packets in the same flow, without $[prodname] rechecking every packet. + +This can, however, make it look like a $[prodname] policy is not working as it +should, if policy is changed to disallow a flow that was previously allowed. +If packets were recently exchanged on the previously allowed flow, and so there +is conntrack state for that flow that has not yet expired, that conntrack state +will allow further packets between the same IP addresses and ports, even after +the $[prodname] policy has been changed. + +Per $[prodname]'s current implementation, there are two workarounds for this: + +- Somehow ensure that no further packets flow between the relevant IP + addresses and ports until the conntrack state has expired (typically about + a minute). + +- Use the 'conntrack' tool to delete the relevant conntrack state; for example + `conntrack -D -p tcp --orig-port-dst 80`. + +Then you should observe that the new $[prodname] policy is enforced for new packets. diff --git a/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/failsafe.mdx b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/failsafe.mdx new file mode 100644 index 0000000000..4165b80489 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/failsafe.mdx @@ -0,0 +1,39 @@ +--- +description: Reference for the Calico Cloud failsafe inbound and outbound port lists that prevent host network policy from cutting off control-plane connectivity. +--- + +# Failsafe rules + +To avoid completely cutting off a host via incorrect or malformed +policy, $[prodname] has a failsafe mechanism that keeps various pinholes open +in the firewall. + +By default, $[prodname] keeps the following ports open on _all_ host endpoints: + +| Port | Protocol | Direction | Purpose | +| ---- | -------- | ------------------ | ------------------------------ | +| 22 | TCP | Inbound | SSH access | +| 53 | UDP | Outbound | DNS queries | +| 67 | UDP | Outbound | DHCP access | +| 68 | UDP | Inbound | DHCP access | +| 179 | TCP | Inbound & Outbound | BGP access (Calico networking) | +| 6443 | TCP | Inbound & Outbound | Kubernetes API server access | + +The lists of failsafe ports can be configured via the configuration parameters +`FailsafeInboundHostPorts` and `FailsafeOutboundHostPorts` +described in [Configuring Felix](../component-resources/node/felix/configuration.mdx) +. They +can be disabled by setting each configuration value to "[]". + +:::note + +Removing the inbound failsafe rules can leave a host inaccessible. + +Removing the outbound failsafe rules can leave Felix unable to connect +to the datastore. + +Before disabling the failsafe rules, we recommend creating a policy to +replace it with more-specific rules for your environment: see +[Creating policy for basic connectivity](connectivity.mdx). + +::: diff --git a/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/forwarded.mdx b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/forwarded.mdx new file mode 100644 index 0000000000..3aea302f01 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/forwarded.mdx @@ -0,0 +1,80 @@ +--- +description: Reference for the applyOnForward field on Calico Cloud host endpoint policy that controls how rules apply to forwarded traffic in connected clusters. +--- + +# Apply on forwarded traffic + +If `applyOnForward` is `false`, the host endpoint policy applies to traffic to/from +local processes only. + +If `applyOnForward` is `true`, the host endpoint policy also applies to forwarded traffic: + +- Traffic that comes in via a host endpoint and is forwarded to a local workload (container/pod/VM). +- Traffic from a local workload that is forwarded out via a host endpoint. +- Traffic that comes in via a host endpoint and is forwarded out via another host endpoint. + +By default, `applyOnForward` is `false`. + +Untracked policies and pre-DNAT policies must have `applyOnForward` set to `true` +because they apply to all forwarded traffic. + +Forwarded traffic is allowed by default if no policies apply to the endpoint and direction. In +other words, if a host endpoint is configured, but there are no policies with `applyOnForward` +set to `true` that apply to that host endpoint and traffic direction, forwarded traffic is +allowed in that direction. For example if a forwarded flow is incoming via a host endpoint, but there are +no Ingress policies with `applyOnForward: true` that apply to that host endpoint, the flow is +allowed. If there are `applyOnForward: true` policies that select the host endpoint and direction, +but no rules in the policies allow the traffic, the traffic is denied. + +This is different from how $[prodname] treats traffic to or from a local process: +if a host endpoint is configured and there are no policies that select the host endpoint in +the traffic direction, or no rules that allow the traffic, the traffic is denied. + +Traffic that traverses a host endpoint and is forwarded to a workload endpoint must also pass +the applicable workload endpoint policy, if any. That is to say, if an `applyOnForward: true` host +endpoint policy allows the traffic, but workload endpoint policy denies it, the packet is still dropped. + +Traffic that ingresses one host endpoint, is forwarded, and egresses host endpoint must +pass ingress policy on the first host endpoint and egress policy on the second host endpoint. + +:::note + +$[prodname]'s handling of host endpoint policy has changed, since before +Calico v3.0, in two ways: + +- It will not apply at all to forwarded traffic, by default. If you have an existing + policy and you want it to apply to forwarded traffic, you need to add `applyOnForward: true` to the policy. +- Even with `applyOnForward: true`, the treatment is not quite the same in + Calico v3.0 as in previous releases, because–once a host endpoint is configured– + Calico v3.0 allows forwarded traffic through that endpoint by default, whereas + previous releases denied forwarded traffic through that endpoint by default. + If you want to maintain the default-deny behavior for all host-endpoint forwarded + traffic, you can create an empty policy with `applyOnForward` set to `true` + that applies to all traffic on all host endpoints. + +::: + +```bash +calicoctl apply -f - < diff --git a/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/objects.mdx b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/objects.mdx new file mode 100644 index 0000000000..870e296a6c --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/objects.mdx @@ -0,0 +1,124 @@ +--- +description: Reference for the HostEndpoint object in Calico Cloud describing how to model a host network interface in a connected cluster so policy can select it. +--- + +# Creating host endpoint objects + +For each host endpoint that you want $[prodname] to secure, you'll need to +create a host endpoint object in etcd. Use the `calicoctl create` command +to create a host endpoint resource (`HostEndpoint`). + +There are two ways to specify the interface that a host endpoint should +refer to. You can either specify the name of the interface or its +expected IP address. In either case, you'll also need to know the name given to +the $[prodname] node running on the host that owns the interface; in most cases this +will be the same as the hostname of the host. + +For example, to secure the interface named `eth0` with IP 10.0.0.1 on +host `my-host`, run the command below. The name of the endpoint is an +arbitrary name required for endpoint identification. + +When running this command, replace the placeholders in angle brackets with +appropriate values for your deployment. + +```bash +calicoctl create -f - < + labels: + role: webserver + environment: production + spec: + interfaceName: eth0 + node: + profiles: [] + expectedIPs: ["10.0.0.1"] +EOF +``` + +:::note + +Felix tries to detect the correct hostname for a system. It logs +out the value it has determined at start-of-day in the following +format: +`2015-10-20 17:42:09,813 \[INFO\]\[30149/5\] calico.felix.config 285: Parameter FelixHostname (Felix compute host hostname) has value 'my-hostname' read from None` +The value (in this case `'my-hostname'`) needs to match the hostname +used in etcd. Ideally, the host's system hostname should be set +correctly but if that's not possible, the Felix value can be +overridden with the FelixHostname configuration setting. See +configuration for more details. + +::: + +Where `` is an optional list of security profiles +to apply to the endpoint and labels contains a set of arbitrary +key/value pairs that can be used in selector expressions. + +{/* TODO(smc) data-model: Link to new data model docs. */} + +:::note + +When rendering security rules on other hosts, $[prodname] uses the +`expectedIPs` field to resolve label selectors +to IP addresses. If the `expectedIPs` field is omitted +then security rules that use labels will fail to match +this endpoint. +Or, if you knew that the IP address should be 10.0.0.1, but not the name +of the interface: + +::: + +```bash +calicoctl create -f - < + labels: + role: webserver + environment: production + spec: + node: + profiles: [] + expectedIPs: ["10.0.0.1"] +EOF +``` + +After you create host endpoint objects, Felix will start policing +traffic to/from that interface. If you have no policy or profiles in +place, then you should see traffic being dropped on the interface. + +:::note + +By default, $[prodname] has a failsafe in place that allows certain +traffic such as ssh. See below for more details on +disabling/configuring the failsafe rules. + +::: + +If you don't see traffic being dropped, check the hostname, IP address +and (if used) the interface name in the configuration. If there was +something wrong with the endpoint data, Felix will log a validation +error at `WARNING` level and it will ignore the endpoint: + +A `grep` through the Felix logs for the string "Validation failed" should allow +you to locate the error. + +```bash +grep "Validation failed" /var/log/calico/felix.log +``` + +An example error follows. + +``` +2016-05-31 12:16:21,651 [WARNING][8657/3] calico.felix.fetcd 1017: + Validation failed for host endpoint HostEndpointId, treating as + missing: 'name' or 'expected_ipvX_addrs' must be present.; + '{ "labels": {"foo": "bar"}, "profile_ids": ["prof1"]}' +``` + +The error can be quite long but it should log the precise cause of the +rejection; in this case `'name' or 'expected\_ipvX\_addrs' must be present` tells us that either the interface's name or its expected IP +address must be specified. diff --git a/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/overview.mdx b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/overview.mdx new file mode 100644 index 0000000000..7880faa098 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/overview.mdx @@ -0,0 +1,57 @@ +--- +description: Reference overview of host endpoint protection in Calico Cloud covering the model for securing host network interfaces with policy across connected clusters. +--- + +# Host endpoints + +This guide describes how to use $[prodname] to secure the network interfaces +of the host itself (as opposed to those of any container/VM workloads +that are present on the host). We call such interfaces "host endpoints", +to distinguish them from "workload endpoints" (such as containers or VMs). + +$[prodname] supports the same rich security policy model for host endpoints (host +endpoint policy) that it supports for workload endpoints. Host endpoints can +have labels, and their labels are in the same "namespace" as those of workload +endpoints. This allows security rules for either type of endpoint to refer to +the other type (or a mix of the two) using labels and selectors. + +$[prodname] does not support setting IPs or policing MAC addresses for host +interfaces, it assumes that the interfaces are configured by the +underlying network fabric. + +$[prodname] distinguishes workload endpoints from host endpoints by a configurable +prefix. Unless you happen to have host interfaces whose name matches the +default for that prefix (`cali`), you won't need to change it. In case you do, +see the `InterfacePrefix` configuration value at [Configuring Felix](../component-resources/node/felix/configuration.mdx) +. +Interfaces that start with a value listed in `InterfacePrefix` are assumed to +be workload interfaces. Others are treated as host interfaces. + +$[prodname] blocks all traffic to/from workload interfaces by default; +allowing traffic only if the interface is known and policy is in place. +However, for host endpoints, $[prodname] is more lenient; it only polices +traffic to/from interfaces that it's been explicitly told about. Traffic +to/from other interfaces is left alone. + +You can use host endpoint policy to secure a NAT gateway or router. $[prodname] +supports selector-based policy when running on a gateway or router, allowing for +rich, dynamic security policy based on the labels attached to your host endpoints. + +You can apply host endpoint policies to three types of traffic: + +- Traffic that is terminated locally. +- Traffic that is forwarded between host endpoints. +- Traffic that is forwarded between a host endpoint and a workload endpoint on the + same host. + +Set the `applyOnForward` flag to `true` to apply a policy to forwarded traffic. +See [GlobalNetworkPolicy spec](../resources/globalnetworkpolicy.mdx#spec). + +:::note + +Both traffic forwarded between host endpoints and traffic forwarded +between a host endpoint and a workload endpoint on the same host is regarded as +`forwarded traffic`. +![Diagram showing packet flows for forwarded traffic between host endpoints and between host endpoints and workload endpoints](/img/calico-enterprise/bare-metal-packet-flows.svg) + +::: diff --git a/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/pre-dnat.mdx b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/pre-dnat.mdx new file mode 100644 index 0000000000..2727edd604 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/pre-dnat.mdx @@ -0,0 +1,46 @@ +--- +description: Reference for pre-DNAT host endpoint policy in Calico Cloud connected clusters that applies rules to ingress traffic before destination NAT rewrites the address. +--- + +# Pre-DNAT policy + +Policy for host endpoints can be marked as `preDNAT`. This means that rules in +that policy should be applied before any DNAT (Destination Network Address +Translation), which is useful if it is more convenient to specify $[prodname] policy +in terms of a packet's original destination IP address and port, than in terms +of that packet's destination IP address and port after it has been DNAT'd. + +An example is securing access to Kubernetes NodePorts from outside the cluster. +Traffic from outside is addressed to any node's IP address, on a known +NodePort, and Kubernetes (kube-proxy) then DNATs that to the IP address of one +of the pods that provides the corresponding service, and the relevant port +number on that pod (which is usually different from the NodePort). + +As NodePorts are the externally advertised way of connecting to services (and a +NodePort uniquely identifies a service, whereas an internal port number may +not), it makes sense to express $[prodname] policy to expose or secure particular +Services in terms of the corresponding NodePorts. But that is only possible if +the $[prodname] policy is applied before DNAT changes the NodePort to something +else. Hence this kind of policy needs `preDNAT` set to `true`. + +In addition to being applied before any DNAT, the enforcement of pre-DNAT +policy differs from that of normal host endpoint policy in three key details, +reflecting that it is designed for the policing of incoming traffic from +outside the cluster: + +- Pre-DNAT policy may only have ingress rules, not egress. (When incoming + traffic is allowed by the ingress rules, standard connection tracking is + sufficient to allow the return path traffic.) + +- Pre-DNAT policy is enforced for all traffic arriving through a host + endpoint, regardless of where that traffic is going, and - in particular - + even if that traffic is routed to a local workload on the same host. + (Whereas normal host endpoint policy is skipped, for traffic going to a + local workload.) + +- There is no 'default drop' semantic for pre-DNAT policy (as there is for + normal host endpoint policy). In other words, if a host endpoint is defined + but has no pre-DNAT policies that explicitly allow or deny a particular + incoming packet, that packet is allowed to continue on its way, and will + then be accepted or dropped according to workload policy (if it is going to + a local workload) or to normal host endpoint policy (if not). diff --git a/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/selector.mdx b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/selector.mdx new file mode 100644 index 0000000000..ea8a0536cd --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/host-endpoints/selector.mdx @@ -0,0 +1,30 @@ +--- +description: Reference for ordered host endpoint policies in Calico Cloud connected clusters that match interfaces using label selectors. +--- + +# Selector-based policies + +We recommend using selector-based security policy with +host endpoints. This allows ordered policy to be applied to +endpoints that match particular label selectors. + +For example, you could add a second policy for webserver access: + +```bash +cat < + + + + + +## Resource definitions + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +## Component resources + + + + + + + + + + +## Configuration on public clouds + + + + + + + +## Host endpoints + + + + + + + + + + + + + + +## Architecture + + + + + + + + + +## Other reference topics + + + + + + + + diff --git a/calico-cloud_versioned_docs/version-24-1/reference/installation/_README.mdx b/calico-cloud_versioned_docs/version-24-1/reference/installation/_README.mdx new file mode 100644 index 0000000000..a35570764b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/installation/_README.mdx @@ -0,0 +1,7 @@ +# Generating API reference docs + +The api.html doc in this directory is generated using https://github.com/tmjd/gen-crd-api-reference-docs/tree/kb_v2. + +To generate an updated file, change to the root of the docs repository and run +the appropriate Makefile target. See the `README.md` file for more details on +how to list available targets and which ones to run. \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/reference/installation/_api.mdx b/calico-cloud_versioned_docs/version-24-1/reference/installation/_api.mdx new file mode 100644 index 0000000000..af43be7928 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/installation/_api.mdx @@ -0,0 +1,4993 @@ +Packages +- [operator.tigera.io/v1](#operatortigeraiov1) + + +{/* vale off */} + +## operator.tigera.io/v1 + +API Schema definitions for configuring the installation of Calico and Calico Enterprise + +Package v1 contains API Schema definitions for the operator v1 API group + +Resource Types +- [APIServer](#apiserver) +- [ApplicationLayer](#applicationlayer) +- [EgressGateway](#egressgateway) +- [GatewayAPI](#gatewayapi) +- [Goldmane](#goldmane) +- [ImageSet](#imageset) +- [Installation](#installation) +- [IntrusionDetection](#intrusiondetection) +- [Istio](#istio) +- [LogCollector](#logcollector) +- [ManagementClusterConnection](#managementclusterconnection) +- [Monitor](#monitor) +- [NonClusterHost](#nonclusterhost) +- [PacketCaptureAPI](#packetcaptureapi) +- [PolicyRecommendation](#policyrecommendation) +- [TigeraStatus](#tigerastatus) +- [Whisker](#whisker) + + + +### APIServer + + + +APIServer installs the Tigera API server and related resources. At most one instance +of this resource is supported. It must be named "default" or "tigera-secure". + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `APIServer` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[APIServerSpec](#apiserverspec)_ | Specification of the desired state for the Tigera API server. | +| `status` _[APIServerStatus](#apiserverstatus)_ | Most recently observed status for the Tigera API server. | + + +### APIServerDeployment + + + +APIServerDeployment is the configuration for the API server Deployment. + +_Appears in:_ +- [APIServerSpec](#apiserverspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[APIServerDeploymentSpec](#apiserverdeploymentspec)_ | (Optional) Spec is the specification of the API server Deployment. | + + +### APIServerDeploymentContainer + + + +APIServerDeploymentContainer is an API server Deployment container. + +_Appears in:_ +- [APIServerDeploymentPodSpec](#apiserverdeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the API server Deployment container by name.
Supported values are: calico-apiserver, tigera-queryserver, calico-l7-admission-controller | +| `ports` _[APIServerDeploymentContainerPort](#apiserverdeploymentcontainerport) array_ | (Optional) Ports allows customization of container's ports. If specified, this overrides the named APIServer Deployment container's ports. If omitted, the API server Deployment will use its default value for this container's port. | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named API server Deployment container's resources. If omitted, the API server Deployment will use its default value for this container's resources. If used in conjunction with the deprecated ComponentResources, then this value takes precedence. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### APIServerDeploymentContainerPort + + + + + +_Appears in:_ +- [APIServerDeploymentContainer](#apiserverdeploymentcontainer) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the API server Deployment Container port by name.
Supported values are: apiserver, queryserver, l7admctrl | +| `containerPort` _integer_ | Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. | + + +### APIServerDeploymentInitContainer + + + +APIServerDeploymentInitContainer is an API server Deployment init container. + +_Appears in:_ +- [APIServerDeploymentPodSpec](#apiserverdeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the API server Deployment init container by name.
Supported values are: calico-apiserver-certs-key-cert-provisioner | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named API server Deployment init container's resources. If omitted, the API server Deployment will use its default value for this init container's resources. | + + +### APIServerDeploymentPodSpec + + + +APIServerDeploymentDeploymentPodSpec is the API server Deployment's PodSpec. + +_Appears in:_ +- [APIServerDeploymentPodTemplateSpec](#apiserverdeploymentpodtemplatespec) + +| Field | Description | +| --- | --- | +| `initContainers` _[APIServerDeploymentInitContainer](#apiserverdeploymentinitcontainer) array_ | (Optional) InitContainers is a list of API server init containers. If specified, this overrides the specified API server Deployment init containers. If omitted, the API server Deployment will use its default values for its init containers. | +| `containers` _[APIServerDeploymentContainer](#apiserverdeploymentcontainer) array_ | (Optional) Containers is a list of API server containers. If specified, this overrides the specified API server Deployment containers. If omitted, the API server Deployment will use its default values for its containers. | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) Affinity is a group of affinity scheduling rules for the API server pods. If specified, this overrides any affinity that may be set on the API server Deployment. If omitted, the API server Deployment will use its default value for affinity.
WARNING: Please note that this field will override the default API server Deployment affinity. | +| `nodeSelector` _object (keys:string, values:string)_ | NodeSelector is the API server pod's scheduling constraints. If specified, each of the key/value pairs are added to the API server Deployment nodeSelector provided the key does not already exist in the object's nodeSelector. If used in conjunction with ControlPlaneNodeSelector, that nodeSelector is set on the API server Deployment and each of this field's key/value pairs are added to the API server Deployment nodeSelector provided the key does not already exist in the object's nodeSelector. If omitted, the API server Deployment will use its default value for nodeSelector.
WARNING: Please note that this field will modify the default API server Deployment nodeSelector. | +| `topologySpreadConstraints` _[TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#topologyspreadconstraint-v1-core) array_ | (Optional) TopologySpreadConstraints describes how a group of pods ought to spread across topology domains. Scheduler will schedule pods in a way which abides by the constraints. All topologySpreadConstraints are ANDed. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) Tolerations is the API server pod's tolerations. If specified, this overrides any tolerations that may be set on the API server Deployment. If omitted, the API server Deployment will use its default value for tolerations.
WARNING: Please note that this field will override the default API server Deployment tolerations. | +| `priorityClassName` _string_ | (Optional) PriorityClassName allows to specify a PriorityClass resource to be used. | + + +### APIServerDeploymentPodTemplateSpec + + + +APIServerDeploymentPodTemplateSpec is the API server Deployment's PodTemplateSpec + +_Appears in:_ +- [APIServerDeploymentSpec](#apiserverdeploymentspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[APIServerDeploymentPodSpec](#apiserverdeploymentpodspec)_ | (Optional) Spec is the API server Deployment's PodSpec. | + + +### APIServerDeploymentSpec + + + +APIServerDeploymentSpec defines configuration for the API server Deployment. + +_Appears in:_ +- [APIServerDeployment](#apiserverdeployment) + +| Field | Description | +| --- | --- | +| `minReadySeconds` _integer_ | (Optional) MinReadySeconds is the minimum number of seconds for which a newly created Deployment pod should be ready without any of its container crashing, for it to be considered available. If specified, this overrides any minReadySeconds value that may be set on the API server Deployment. If omitted, the API server Deployment will use its default value for minReadySeconds. | +| `template` _[APIServerDeploymentPodTemplateSpec](#apiserverdeploymentpodtemplatespec)_ | (Optional) Template describes the API server Deployment pod that will be created. | + + +### APIServerLogging + + + + + +_Appears in:_ +- [APIServerPodLogging](#apiserverpodlogging) + +| Field | Description | +| --- | --- | +| `logSeverity` _[LogSeverity](#logseverity)_ | (Optional) LogSeverity defines log level for APIServer container. | + + +### APIServerPodLogging + + + + + +_Appears in:_ +- [APIServerSpec](#apiserverspec) + +| Field | Description | +| --- | --- | +| `apiServer` _[APIServerLogging](#apiserverlogging)_ | (Optional) | +| `queryServer` _[QueryServerLogging](#queryserverlogging)_ | (Optional) | + + +### APIServerSpec + + + +APIServerSpec defines the desired state of Tigera API server. + +_Appears in:_ +- [APIServer](#apiserver) + +| Field | Description | +| --- | --- | +| `logging` _[APIServerPodLogging](#apiserverpodlogging)_ | (Optional) | +| `apiServerDeployment` _[APIServerDeployment](#apiserverdeployment)_ | APIServerDeployment configures the calico-apiserver Deployment. If used in conjunction with ControlPlaneNodeSelector or ControlPlaneTolerations, then these overrides take precedence. | +| `calicoWebhooksDeployment` _[CalicoWebhooksDeployment](#calicowebhooksdeployment)_ | (Optional) CalicoWebhooksDeployment configures the calico-webhooks Deployment. | + + +### APIServerStatus + + + +APIServerStatus defines the observed state of Tigera API server. + +_Appears in:_ +- [APIServer](#apiserver) + +| Field | Description | +| --- | --- | +| `state` _string_ | State provides user-readable status. | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#condition-v1-meta) array_ | (Optional) Conditions represents the latest observed set of conditions for the component. A component may be one or more of Ready, Progressing, Degraded or other customer types. | + + +### AWSEgressGateway + + + +AWSEgressGateway defines the configurations for deploying EgressGateway in AWS + +_Appears in:_ +- [EgressGatewaySpec](#egressgatewayspec) + +| Field | Description | +| --- | --- | +| `nativeIP` _[NativeIP](#nativeip)_ | (Optional) NativeIP defines if EgressGateway is to use an AWS backed IPPool.
Default: Disabled | +| `elasticIPs` _string array_ | (Optional) ElasticIPs defines the set of elastic IPs that can be used for Egress Gateway pods. NativeIP must be Enabled if elastic IPs are set. | + + +### AdditionalLogSourceSpec + + + + + +_Appears in:_ +- [LogCollectorSpec](#logcollectorspec) + +| Field | Description | +| --- | --- | +| `eksCloudwatchLog` _[EksCloudwatchLogsSpec](#ekscloudwatchlogsspec)_ | (Optional) If specified with EKS Provider in Installation, enables fetching EKS audit logs. | + + +### AdditionalLogStoreSpec + + + + + +_Appears in:_ +- [LogCollectorSpec](#logcollectorspec) + +| Field | Description | +| --- | --- | +| `s3` _[S3StoreSpec](#s3storespec)_ | (Optional) If specified, enables exporting of flow, audit, and DNS logs to Amazon S3 storage. | +| `syslog` _[SyslogStoreSpec](#syslogstorespec)_ | (Optional) If specified, enables exporting of flow, audit, and DNS logs to syslog. | +| `splunk` _[SplunkStoreSpec](#splunkstorespec)_ | (Optional) If specified, enables exporting of flow, audit, and DNS logs to splunk. | + + +### Alertmanager + + + + + +_Appears in:_ +- [MonitorSpec](#monitorspec) + +| Field | Description | +| --- | --- | +| `spec` _[AlertmanagerSpec](#alertmanagerspec)_ | (Optional) Spec is the specification of the Alertmanager. | + + +### AlertmanagerSpec + + + + + +_Appears in:_ +- [Alertmanager](#alertmanager) + +| Field | Description | +| --- | --- | +| `replicas` _integer_ | (Optional) Replicas defines the number of Alertmanager replicas. When set to 0, Alertmanager is not rendered.
Default: 0 | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | Define resources requests and limits for single Pods. | + + +### AnomalyDetectionSpec + + + + + +_Appears in:_ +- [IntrusionDetectionSpec](#intrusiondetectionspec) + +| Field | Description | +| --- | --- | +| `storageClassName` _string_ | (Optional) StorageClassName is now deprecated, and configuring it has no effect. | + + +### ApplicationLayer + + + +ApplicationLayer is the Schema for the applicationlayers API + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `ApplicationLayer` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[ApplicationLayerSpec](#applicationlayerspec)_ | | +| `status` _[ApplicationLayerStatus](#applicationlayerstatus)_ | | + + +### ApplicationLayerPolicyStatusType + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [Enabled Disabled] + + +_Appears in:_ +- [ApplicationLayerSpec](#applicationlayerspec) + +| Value | Description | +| --- | --- | +| `Enabled` | | +| `Disabled` | | + + +### ApplicationLayerSpec + + + +ApplicationLayerSpec defines the desired state of ApplicationLayer + +_Appears in:_ +- [ApplicationLayer](#applicationlayer) + +| Field | Description | +| --- | --- | +| `webApplicationFirewall` _[WAFStatusType](#wafstatustype)_ | WebApplicationFirewall controls whether or not ModSecurity enforcement is enabled for the cluster. When enabled, Services may opt-in to having ingress traffic examed by ModSecurity. | +| `logCollection` _[LogCollectionSpec](#logcollectionspec)_ | Specification for application layer (L7) log collection. | +| `applicationLayerPolicy` _[ApplicationLayerPolicyStatusType](#applicationlayerpolicystatustype)_ | Application Layer Policy controls whether or not ALP enforcement is enabled for the cluster. When enabled, NetworkPolicies with HTTP Match rules may be defined to opt-in workloads for traffic enforcement on the application layer. | +| `envoy` _[EnvoySettings](#envoysettings)_ | User-configurable settings for the Envoy proxy. | +| `l7LogCollectorDaemonSet` _[L7LogCollectorDaemonSet](#l7logcollectordaemonset)_ | (Optional) L7LogCollectorDaemonSet configures the L7LogCollector DaemonSet. | +| `sidecarInjection` _[SidecarStatusType](#sidecarstatustype)_ | (Optional) SidecarInjection controls whether or not sidecar injection is enabled for the cluster. When enabled, pods with the label "applicationlayer.projectcalico.org/sidecar"="true" will have their L7 functionality such as WAF and ALP implemented using an injected sidecar instead of a per-host proxy. The per-host proxy will continue to be used for pods without this label. | + + +### ApplicationLayerStatus + + + +ApplicationLayerStatus defines the observed state of ApplicationLayer + +_Appears in:_ +- [ApplicationLayer](#applicationlayer) + +| Field | Description | +| --- | --- | +| `state` _string_ | State provides user-readable status. | +| `sidecarWebhook` _[SidecarWebhookStateType](#sidecarwebhookstatetype)_ | SidecarWebhook provides the state of sidecar injection mutatinwebhookconfiguration | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#condition-v1-meta) array_ | (Optional) Conditions represents the latest observed set of conditions for the component. A component may be one or more of Ready, Progressing, Degraded or other customer types. | + + +### AssignmentMode + +_Underlying type:_ _string_ + + + +_Appears in:_ +- [IPPool](#ippool) + +| Value | Description | +| --- | --- | +| `Automatic` | | +| `Manual` | | + + + + +### Azure + + + + + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `policyMode` _[PolicyMode](#policymode)_ | (Optional) PolicyMode determines whether the "control-plane" label is applied to namespaces. It offers two options: Default and Manual. The Default option adds the "control-plane" label to the required namespaces. The Manual option does not apply the "control-plane" label to any namespace.
Default: Default | + + +### BGPOption + +_Underlying type:_ _string_ + +BGPOption describes the mode of BGP to use. + +One of: Enabled, Disabled + +_Appears in:_ +- [CalicoNetworkSpec](#caliconetworkspec) + +| Value | Description | +| --- | --- | +| `Enabled` | | +| `Disabled` | | + + +### BPFNetworkBootstrapType + +_Underlying type:_ _string_ + +BPFNetworkBootstrapType defines how the initial networking configuration is executed. + +_Appears in:_ +- [CalicoNetworkSpec](#caliconetworkspec) + +| Value | Description | +| --- | --- | +| `Enabled` | | +| `Disabled` | | + + +### CAType + +_Underlying type:_ _string_ + +CAType specifies which verification method the tunnel client should use to verify the tunnel server's identity. + +One of: Tigera, Public + +_Appears in:_ +- [ManagementClusterTLS](#managementclustertls) + +| Value | Description | +| --- | --- | +| `Tigera` | | +| `Public` | | + + +### CNIInstallMode + +_Underlying type:_ _string_ + +CNIInstallMode controls which CNI plugin binaries the operator installs onto the host. + +_Appears in:_ +- [CNISpec](#cnispec) + +| Value | Description | +| --- | --- | +| `All` | CNIInstallModeAll installs Calico's own CNI binaries plus the upstream plugin set (host-local, portmap, loopback, tuning, flannel) via a dedicated init container. | +| `CalicoOnly` | CNIInstallModeCalicoOnly installs only Calico's own CNI binaries; the host is expected to provide any required upstream plugins. | + + +### CNILogging + + + + + +_Appears in:_ +- [Logging](#logging) + +| Field | Description | +| --- | --- | +| `logSeverity` _[LogLevel](#loglevel)_ | (Optional)
Default: Info | +| `logFileMaxSize` _[Quantity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#quantity-resource-api)_ | (Optional)
Default: 100Mi | +| `logFileMaxAgeDays` _integer_ | (Optional)
Default: 30 (days) | +| `logFileMaxCount` _integer_ | (Optional)
Default: 10 | + + +### CNIPluginType + +_Underlying type:_ _string_ + +CNIPluginType describes the type of CNI plugin used. + +One of: Calico, GKE, AmazonVPC, AzureVNET + +_Appears in:_ +- [CNISpec](#cnispec) + +| Value | Description | +| --- | --- | +| `Calico` | | +| `GKE` | | +| `AmazonVPC` | | +| `AzureVNET` | | + + +### CNISpec + + + +CNISpec contains configuration for the CNI plugin. + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `type` _[CNIPluginType](#cniplugintype)_ | Specifies the CNI plugin that will be used in the Calico or Calico Enterprise installation. * For KubernetesProvider GKE, this field defaults to GKE. * For KubernetesProvider AKS, this field defaults to AzureVNET. * For KubernetesProvider EKS, this field defaults to AmazonVPC. * If aws-node daemonset exists in kube-system when the Installation resource is created, this field defaults to AmazonVPC. * For all other cases this field defaults to Calico. For the value Calico, the CNI plugin binaries and CNI config will be installed as part of deployment, for all other values the CNI plugin binaries and CNI config is a dependency that is expected to be installed separately.
Default: Calico | +| `ipam` _[IPAMSpec](#ipamspec)_ | (Optional) IPAM specifies the pod IP address management that will be used in the Calico or Calico Enterprise installation. | +| `binDir` _string_ | (Optional) BinDir is the path to the CNI binaries directory. If you have changed the installation directory for CNI binaries in the container runtime configuration, please ensure that this field points to the same directory as specified in the container runtime settings. Default directory depends on the KubernetesProvider. * For KubernetesProvider GKE, this field defaults to "/home/kubernetes/bin". * For KubernetesProvider OpenShift, this field defaults to "/var/lib/cni/bin". * Otherwise, this field defaults to "/opt/cni/bin". | +| `confDir` _string_ | (Optional) ConfDir is the path to the CNI config directory. If you have changed the installation directory for CNI configuration in the container runtime configuration, please ensure that this field points to the same directory as specified in the container runtime settings. Default directory depends on the KubernetesProvider. * For KubernetesProvider GKE, this field defaults to "/etc/cni/net.d". * For KubernetesProvider OpenShift, this field defaults to "/var/run/multus/cni/net.d". * Otherwise, this field defaults to "/etc/cni/net.d". | +| `installMode` _[CNIInstallMode](#cniinstallmode)_ | (Optional) InstallMode controls which CNI plugin binaries the operator installs onto each node when CNI.Type is Calico. * All (default): the operator runs a cni-plugins init container that stages upstream CNI plugin binaries (host-local, portmap, loopback, tuning, flannel) into a shared volume, and the install-cni init container copies them onto the host alongside Calico's own binaries. * CalicoOnly: skip the cni-plugins init container. Only Calico's own binaries are installed. Use this when the host already provides the upstream plugins (e.g. kind, certain managed node images).
Default: All | + + +### CRDManagement + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [Reconcile PreferExisting] + + +_Appears in:_ +- [GatewayAPISpec](#gatewayapispec) + +| Value | Description | +| --- | --- | +| `Reconcile` | | +| `PreferExisting` | | + + +### CSINodeDriverDaemonSet + + + +CSINodeDriverDaemonSet is the configuration for the csi-node-driver DaemonSet. + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CSINodeDriverDaemonSetSpec](#csinodedriverdaemonsetspec)_ | (Optional) Spec is the specification of the csi-node-driver DaemonSet. | + + +### CSINodeDriverDaemonSetContainer + + + +CSINodeDriverDaemonSetContainer is a csi-node-driver DaemonSet container. + +_Appears in:_ +- [CSINodeDriverDaemonSetPodSpec](#csinodedriverdaemonsetpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the csi-node-driver DaemonSet container by name.
Supported values are: calico-csi, csi-node-driver-registrar. | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named csi-node-driver DaemonSet container's resources. If omitted, the csi-node-driver DaemonSet will use its default value for this container's resources. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### CSINodeDriverDaemonSetPodSpec + + + +CSINodeDriverDaemonSetPodSpec is the csi-node-driver DaemonSet's PodSpec. + +_Appears in:_ +- [CSINodeDriverDaemonSetPodTemplateSpec](#csinodedriverdaemonsetpodtemplatespec) + +| Field | Description | +| --- | --- | +| `containers` _[CSINodeDriverDaemonSetContainer](#csinodedriverdaemonsetcontainer) array_ | (Optional) Containers is a list of csi-node-driver containers. If specified, this overrides the specified csi-node-driver DaemonSet containers. If omitted, the csi-node-driver DaemonSet will use its default values for its containers. | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) Affinity is a group of affinity scheduling rules for the csi-node-driver pods. If specified, this overrides any affinity that may be set on the csi-node-driver DaemonSet. If omitted, the csi-node-driver DaemonSet will use its default value for affinity.
WARNING: Please note that this field will override the default csi-node-driver DaemonSet affinity. | +| `nodeSelector` _object (keys:string, values:string)_ | (Optional) NodeSelector is the csi-node-driver pod's scheduling constraints. If specified, each of the key/value pairs are added to the csi-node-driver DaemonSet nodeSelector provided the key does not already exist in the object's nodeSelector. If omitted, the csi-node-driver DaemonSet will use its default value for nodeSelector.
WARNING: Please note that this field will modify the default csi-node-driver DaemonSet nodeSelector. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) Tolerations is the csi-node-driver pod's tolerations. If specified, this overrides any tolerations that may be set on the csi-node-driver DaemonSet. If omitted, the csi-node-driver DaemonSet will use its default value for tolerations.
WARNING: Please note that this field will override the default csi-node-driver DaemonSet tolerations. | + + +### CSINodeDriverDaemonSetPodTemplateSpec + + + +CSINodeDriverDaemonSetPodTemplateSpec is the csi-node-driver DaemonSet's PodTemplateSpec + +_Appears in:_ +- [CSINodeDriverDaemonSetSpec](#csinodedriverdaemonsetspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CSINodeDriverDaemonSetPodSpec](#csinodedriverdaemonsetpodspec)_ | (Optional) Spec is the csi-node-driver DaemonSet's PodSpec. | + + +### CSINodeDriverDaemonSetSpec + + + +CSINodeDriverDaemonSetSpec defines configuration for the csi-node-driver DaemonSet. + +_Appears in:_ +- [CSINodeDriverDaemonSet](#csinodedriverdaemonset) + +| Field | Description | +| --- | --- | +| `minReadySeconds` _integer_ | (Optional) MinReadySeconds is the minimum number of seconds for which a newly created DaemonSet pod should be ready without any of its container crashing, for it to be considered available. If specified, this overrides any minReadySeconds value that may be set on the csi-node-driver DaemonSet. If omitted, the csi-node-driver DaemonSet will use its default value for minReadySeconds. | +| `template` _[CSINodeDriverDaemonSetPodTemplateSpec](#csinodedriverdaemonsetpodtemplatespec)_ | (Optional) Template describes the csi-node-driver DaemonSet pod that will be created. | + + +### CalicoKubeControllersDeployment + + + +CalicoKubeControllersDeployment is the configuration for the calico-kube-controllers Deployment. + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CalicoKubeControllersDeploymentSpec](#calicokubecontrollersdeploymentspec)_ | (Optional) Spec is the specification of the calico-kube-controllers Deployment. | + + +### CalicoKubeControllersDeploymentContainer + + + +CalicoKubeControllersDeploymentContainer is a calico-kube-controllers Deployment container. + +_Appears in:_ +- [CalicoKubeControllersDeploymentPodSpec](#calicokubecontrollersdeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the calico-kube-controllers Deployment container by name.
Supported values are: calico-kube-controllers, es-calico-kube-controllers | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named calico-kube-controllers Deployment container's resources. If omitted, the calico-kube-controllers Deployment will use its default value for this container's resources. If used in conjunction with the deprecated ComponentResources, then this value takes precedence. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### CalicoKubeControllersDeploymentPodSpec + + + +CalicoKubeControllersDeploymentPodSpec is the calico-kube-controller Deployment's PodSpec. + +_Appears in:_ +- [CalicoKubeControllersDeploymentPodTemplateSpec](#calicokubecontrollersdeploymentpodtemplatespec) + +| Field | Description | +| --- | --- | +| `containers` _[CalicoKubeControllersDeploymentContainer](#calicokubecontrollersdeploymentcontainer) array_ | (Optional) Containers is a list of calico-kube-controllers containers. If specified, this overrides the specified calico-kube-controllers Deployment containers. If omitted, the calico-kube-controllers Deployment will use its default values for its containers. | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) Affinity is a group of affinity scheduling rules for the calico-kube-controllers pods. If specified, this overrides any affinity that may be set on the calico-kube-controllers Deployment. If omitted, the calico-kube-controllers Deployment will use its default value for affinity.
WARNING: Please note that this field will override the default calico-kube-controllers Deployment affinity. | +| `nodeSelector` _object (keys:string, values:string)_ | NodeSelector is the calico-kube-controllers pod's scheduling constraints. If specified, each of the key/value pairs are added to the calico-kube-controllers Deployment nodeSelector provided the key does not already exist in the object's nodeSelector. If used in conjunction with ControlPlaneNodeSelector, that nodeSelector is set on the calico-kube-controllers Deployment and each of this field's key/value pairs are added to the calico-kube-controllers Deployment nodeSelector provided the key does not already exist in the object's nodeSelector. If omitted, the calico-kube-controllers Deployment will use its default value for nodeSelector.
WARNING: Please note that this field will modify the default calico-kube-controllers Deployment nodeSelector. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) Tolerations is the calico-kube-controllers pod's tolerations. If specified, this overrides any tolerations that may be set on the calico-kube-controllers Deployment. If omitted, the calico-kube-controllers Deployment will use its default value for tolerations.
WARNING: Please note that this field will override the default calico-kube-controllers Deployment tolerations. | + + +### CalicoKubeControllersDeploymentPodTemplateSpec + + + +CalicoKubeControllersDeploymentPodTemplateSpec is the calico-kube-controllers Deployment's PodTemplateSpec + +_Appears in:_ +- [CalicoKubeControllersDeploymentSpec](#calicokubecontrollersdeploymentspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CalicoKubeControllersDeploymentPodSpec](#calicokubecontrollersdeploymentpodspec)_ | (Optional) Spec is the calico-kube-controllers Deployment's PodSpec. | + + +### CalicoKubeControllersDeploymentSpec + + + +CalicoKubeControllersDeploymentSpec defines configuration for the calico-kube-controllers Deployment. + +_Appears in:_ +- [CalicoKubeControllersDeployment](#calicokubecontrollersdeployment) + +| Field | Description | +| --- | --- | +| `minReadySeconds` _integer_ | (Optional) MinReadySeconds is the minimum number of seconds for which a newly created Deployment pod should be ready without any of its container crashing, for it to be considered available. If specified, this overrides any minReadySeconds value that may be set on the calico-kube-controllers Deployment. If omitted, the calico-kube-controllers Deployment will use its default value for minReadySeconds. | +| `template` _[CalicoKubeControllersDeploymentPodTemplateSpec](#calicokubecontrollersdeploymentpodtemplatespec)_ | (Optional) Template describes the calico-kube-controllers Deployment pod that will be created. | + + +### CalicoNetworkSpec + + + +CalicoNetworkSpec specifies configuration options for Calico provided pod networking. + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `linuxDataplane` _[LinuxDataplaneOption](#linuxdataplaneoption)_ | (Optional) LinuxDataplane is used to select the dataplane used for Linux nodes. In particular, it causes the operator to add required mounts and environment variables for the particular dataplane. If not specified, iptables mode is used.
Default: Iptables | +| `windowsDataplane` _[WindowsDataplaneOption](#windowsdataplaneoption)_ | (Optional) WindowsDataplane is used to select the dataplane used for Windows nodes. In particular, it causes the operator to add required mounts and environment variables for the particular dataplane. If not specified, it is disabled and the operator will not render the Calico Windows nodes daemonset.
Default: Disabled | +| `bpfNetworkBootstrap` _[BPFNetworkBootstrapType](#bpfnetworkbootstraptype)_ | (Optional) BPFNetworkBootstrap manages the initial networking setup required to configure the BPF dataplane. When enabled, the operator tries to bootstraps access to the Kubernetes API Server by using the Kubernetes service and its associated endpoints. This field should be enabled only if linuxDataplane is set to "BPF". If another dataplane is selected, this field must be omitted or explicitly set to Disabled. When disabled and linuxDataplane is BPF, you must manually provide the Kubernetes API Server information via the "kubernetes-service-endpoint" ConfigMap. It is invalid to use both the ConfigMap and have this field set to true at the same time.
Default: Disabled | +| `kubeProxyManagement` _[KubeProxyManagementType](#kubeproxymanagementtype)_ | (Optional) KubeProxyManagement controls whether the operator manages the kube-proxy DaemonSet. When enabled, the operator will manage the DaemonSet by patching it: it disables kube-proxy if the dataplane is BPF, or enables it otherwise.
Default: Disabled | +| `bgp` _[BGPOption](#bgpoption)_ | (Optional) BGP configures whether or not to enable Calico's BGP capabilities. | +| `clusterRoutingMode` _[ClusterRoutingMode](#clusterroutingmode)_ | (Optional) ClusterRoutingMode controls how nodes get a route to a workload on another node, when that workload's IP comes from an IP Pool with vxlanMode: Never. When ClusterRoutingMode is BIRD, confd and BIRD program that route. When ClusterRoutingMode is Felix, it is expected that Felix will program that route. Felix always programs such routes for IP Pools with vxlanMode: Always or vxlanMode: CrossSubnet. [Default: BIRD] | +| `ipPools` _[IPPool](#ippool) array_ | (Optional) IPPools contains a list of IP pools to manage. If nil, a single IPv4 IP pool will be created by the operator. If an empty list is provided, the operator will not create any IP pools and will instead wait for IP pools to be created out-of-band. IP pools in this list will be reconciled by the operator and should not be modified out-of-band. | +| `mtu` _integer_ | (Optional) MTU specifies the maximum transmission unit to use on the pod network. If not specified, Calico will perform MTU auto-detection based on the cluster network. | +| `nodeAddressAutodetectionV4` _[NodeAddressAutodetection](#nodeaddressautodetection)_ | (Optional) NodeAddressAutodetectionV4 specifies an approach to automatically detect node IPv4 addresses. If not specified, will use default auto-detection settings to acquire an IPv4 address for each node. | +| `nodeAddressAutodetectionV6` _[NodeAddressAutodetection](#nodeaddressautodetection)_ | (Optional) NodeAddressAutodetectionV6 specifies an approach to automatically detect node IPv6 addresses. If not specified, IPv6 addresses will not be auto-detected. | +| `hostPorts` _[HostPortsType](#hostportstype)_ | (Optional) HostPorts configures whether or not Calico will support Kubernetes HostPorts. Valid only when using the Calico CNI plugin.
Default: Enabled | +| `multiInterfaceMode` _[MultiInterfaceMode](#multiinterfacemode)_ | (Optional) MultiInterfaceMode configures what will configure multiple interface per pod. Only valid for Calico Enterprise installations using the Calico CNI plugin.
Default: None | +| `containerIPForwarding` _[ContainerIPForwardingType](#containeripforwardingtype)_ | (Optional) ContainerIPForwarding configures whether ip forwarding will be enabled for containers in the CNI configuration.
Default: Disabled | +| `linuxPodInterfaceType` _[LinuxPodInterfaceType](#linuxpodinterfacetype)_ | (Optional) LinuxPodInterfaceType selects the virtual device type the Calico CNI plugin creates for each pod's interface on Linux nodes. When set to Netkit, the CNI plugin creates a netkit L2 pair on kernels that support it (Linux 6.7+) and falls back to a veth pair on older kernels. Netkit is recommended for the BPF dataplane, where it allows BPF programs to attach via BPF_NETKIT_PRIMARY for improved throughput and tail-latency under contention; for non-BPF dataplanes it is functionally equivalent to veth. Only valid when using the Calico CNI plugin.
Default: Veth | +| `sysctl` _[Sysctl](#sysctl) array_ | (Optional) Sysctl configures sysctl parameters for tuning plugin | +| `linuxPolicySetupTimeoutSeconds` _integer_ | (Optional) LinuxPolicySetupTimeoutSeconds delays new pods from running containers until their policy has been programmed in the dataplane. The specified delay defines the maximum amount of time that the Calico CNI plugin will wait for policy to be programmed. Only applies to pods created on Linux nodes. * A value of 0 disables pod startup delays.
Default: 0 | + + +### CalicoNodeDaemonSet + + + +CalicoNodeDaemonSet is the configuration for the calico-node DaemonSet. + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CalicoNodeDaemonSetSpec](#caliconodedaemonsetspec)_ | (Optional) Spec is the specification of the calico-node DaemonSet. | + + +### CalicoNodeDaemonSetContainer + + + +CalicoNodeDaemonSetContainer is a calico-node DaemonSet container. + +_Appears in:_ +- [CalicoNodeDaemonSetPodSpec](#caliconodedaemonsetpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the calico-node DaemonSet container by name.
Supported values are: calico-node | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named calico-node DaemonSet container's resources. If omitted, the calico-node DaemonSet will use its default value for this container's resources. If used in conjunction with the deprecated ComponentResources, then this value takes precedence. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### CalicoNodeDaemonSetInitContainer + + + +CalicoNodeDaemonSetInitContainer is a calico-node DaemonSet init container. + +_Appears in:_ +- [CalicoNodeDaemonSetPodSpec](#caliconodedaemonsetpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the calico-node DaemonSet init container by name.
Supported values are: install-cni, cni-plugins, hostpath-init, flexvol-driver, ebpf-bootstrap, node-certs-key-cert-provisioner, calico-node-prometheus-server-tls-key-cert-provisioner, mount-bpffs (deprecated, replaced by ebpf-bootstrap) | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named calico-node DaemonSet init container's resources. If omitted, the calico-node DaemonSet will use its default value for this container's resources. If used in conjunction with the deprecated ComponentResources, then this value takes precedence. | + + +### CalicoNodeDaemonSetPodSpec + + + +CalicoNodeDaemonSetPodSpec is the calico-node DaemonSet's PodSpec. + +_Appears in:_ +- [CalicoNodeDaemonSetPodTemplateSpec](#caliconodedaemonsetpodtemplatespec) + +| Field | Description | +| --- | --- | +| `initContainers` _[CalicoNodeDaemonSetInitContainer](#caliconodedaemonsetinitcontainer) array_ | (Optional) InitContainers is a list of calico-node init containers. If specified, this overrides the specified calico-node DaemonSet init containers. If omitted, the calico-node DaemonSet will use its default values for its init containers. | +| `containers` _[CalicoNodeDaemonSetContainer](#caliconodedaemonsetcontainer) array_ | (Optional) Containers is a list of calico-node containers. If specified, this overrides the specified calico-node DaemonSet containers. If omitted, the calico-node DaemonSet will use its default values for its containers. | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) Affinity is a group of affinity scheduling rules for the calico-node pods. If specified, this overrides any affinity that may be set on the calico-node DaemonSet. If omitted, the calico-node DaemonSet will use its default value for affinity.
WARNING: Please note that this field will override the default calico-node DaemonSet affinity. | +| `nodeSelector` _object (keys:string, values:string)_ | (Optional) NodeSelector is the calico-node pod's scheduling constraints. If specified, each of the key/value pairs are added to the calico-node DaemonSet nodeSelector provided the key does not already exist in the object's nodeSelector. If omitted, the calico-node DaemonSet will use its default value for nodeSelector.
WARNING: Please note that this field will modify the default calico-node DaemonSet nodeSelector. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) Tolerations is the calico-node pod's tolerations. If specified, this overrides any tolerations that may be set on the calico-node DaemonSet. If omitted, the calico-node DaemonSet will use its default value for tolerations.
WARNING: Please note that this field will override the default calico-node DaemonSet tolerations. | +| `dnsPolicy` _[DNSPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#dnspolicy-v1-core)_ | (Optional) DNSPolicy is the DNS policy for the calico-node pods. | +| `dnsConfig` _[PodDNSConfig](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#poddnsconfig-v1-core)_ | (Optional) DNSConfig allows customization of the DNS configuration for the calico-node pods. | + + +### CalicoNodeDaemonSetPodTemplateSpec + + + +CalicoNodeDaemonSetPodTemplateSpec is the calico-node DaemonSet's PodTemplateSpec + +_Appears in:_ +- [CalicoNodeDaemonSetSpec](#caliconodedaemonsetspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CalicoNodeDaemonSetPodSpec](#caliconodedaemonsetpodspec)_ | (Optional) Spec is the calico-node DaemonSet's PodSpec. | + + +### CalicoNodeDaemonSetSpec + + + +CalicoNodeDaemonSetSpec defines configuration for the calico-node DaemonSet. + +_Appears in:_ +- [CalicoNodeDaemonSet](#caliconodedaemonset) + +| Field | Description | +| --- | --- | +| `minReadySeconds` _integer_ | (Optional) MinReadySeconds is the minimum number of seconds for which a newly created DaemonSet pod should be ready without any of its container crashing, for it to be considered available. If specified, this overrides any minReadySeconds value that may be set on the calico-node DaemonSet. If omitted, the calico-node DaemonSet will use its default value for minReadySeconds. | +| `template` _[CalicoNodeDaemonSetPodTemplateSpec](#caliconodedaemonsetpodtemplatespec)_ | (Optional) Template describes the calico-node DaemonSet pod that will be created. | + + +### CalicoNodeWindowsDaemonSet + + + +CalicoNodeWindowsDaemonSet is the configuration for the calico-node-windows DaemonSet. + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CalicoNodeWindowsDaemonSetSpec](#caliconodewindowsdaemonsetspec)_ | (Optional) Spec is the specification of the calico-node-windows DaemonSet. | + + +### CalicoNodeWindowsDaemonSetContainer + + + +CalicoNodeWindowsDaemonSetContainer is a calico-node-windows DaemonSet container. + +_Appears in:_ +- [CalicoNodeWindowsDaemonSetPodSpec](#caliconodewindowsdaemonsetpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the calico-node-windows DaemonSet container by name.
Supported values are: node, felix, confd calico-node-windows is allowed because it was previously allowed. | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named DaemonSet container's resources. If omitted, the DaemonSet will use its default value for this container's resources. If used in conjunction with the deprecated ComponentResources, then this value takes precedence. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### CalicoNodeWindowsDaemonSetInitContainer + + + +CalicoNodeWindowsDaemonSetInitContainer is a calico-node-windows DaemonSet init container. + +_Appears in:_ +- [CalicoNodeWindowsDaemonSetPodSpec](#caliconodewindowsdaemonsetpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the calico-node-windows DaemonSet init container by name.
Supported values are: install-cni;hostpath-init, flexvol-driver, node-certs-key-cert-provisioner, calico-node-windows-prometheus-server-tls-key-cert-provisioner | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named calico-node-windows DaemonSet init container's resources. If omitted, the calico-node-windows DaemonSet will use its default value for this container's resources. If used in conjunction with the deprecated ComponentResources, then this value takes precedence. | + + +### CalicoNodeWindowsDaemonSetPodSpec + + + +CalicoNodeWindowsDaemonSetPodSpec is the calico-node-windows DaemonSet's PodSpec. + +_Appears in:_ +- [CalicoNodeWindowsDaemonSetPodTemplateSpec](#caliconodewindowsdaemonsetpodtemplatespec) + +| Field | Description | +| --- | --- | +| `initContainers` _[CalicoNodeWindowsDaemonSetInitContainer](#caliconodewindowsdaemonsetinitcontainer) array_ | (Optional) InitContainers is a list of calico-node-windows init containers. If specified, this overrides the specified calico-node-windows DaemonSet init containers. If omitted, the calico-node-windows DaemonSet will use its default values for its init containers. | +| `containers` _[CalicoNodeWindowsDaemonSetContainer](#caliconodewindowsdaemonsetcontainer) array_ | (Optional) Containers is a list of calico-node-windows containers. If specified, this overrides the specified calico-node-windows DaemonSet containers. If omitted, the calico-node-windows DaemonSet will use its default values for its containers. | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) Affinity is a group of affinity scheduling rules for the calico-node-windows pods. If specified, this overrides any affinity that may be set on the calico-node-windows DaemonSet. If omitted, the calico-node-windows DaemonSet will use its default value for affinity.
WARNING: Please note that this field will override the default calico-node-windows DaemonSet affinity. | +| `nodeSelector` _object (keys:string, values:string)_ | (Optional) NodeSelector is the calico-node-windows pod's scheduling constraints. If specified, each of the key/value pairs are added to the calico-node-windows DaemonSet nodeSelector provided the key does not already exist in the object's nodeSelector. If omitted, the calico-node-windows DaemonSet will use its default value for nodeSelector.
WARNING: Please note that this field will modify the default calico-node-windows DaemonSet nodeSelector. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) Tolerations is the calico-node-windows pod's tolerations. If specified, this overrides any tolerations that may be set on the calico-node-windows DaemonSet. If omitted, the calico-node-windows DaemonSet will use its default value for tolerations.
WARNING: Please note that this field will override the default calico-node-windows DaemonSet tolerations. | + + +### CalicoNodeWindowsDaemonSetPodTemplateSpec + + + +CalicoNodeWindowsDaemonSetPodTemplateSpec is the calico-node-windows DaemonSet's PodTemplateSpec + +_Appears in:_ +- [CalicoNodeWindowsDaemonSetSpec](#caliconodewindowsdaemonsetspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CalicoNodeWindowsDaemonSetPodSpec](#caliconodewindowsdaemonsetpodspec)_ | (Optional) Spec is the calico-node-windows DaemonSet's PodSpec. | + + +### CalicoNodeWindowsDaemonSetSpec + + + +CalicoNodeWindowsDaemonSetSpec defines configuration for the calico-node-windows DaemonSet. + +_Appears in:_ +- [CalicoNodeWindowsDaemonSet](#caliconodewindowsdaemonset) + +| Field | Description | +| --- | --- | +| `minReadySeconds` _integer_ | (Optional) MinReadySeconds is the minimum number of seconds for which a newly created DaemonSet pod should be ready without any of its container crashing, for it to be considered available. If specified, this overrides any minReadySeconds value that may be set on the calico-node-windows DaemonSet. If omitted, the calico-node-windows DaemonSet will use its default value for minReadySeconds. | +| `template` _[CalicoNodeWindowsDaemonSetPodTemplateSpec](#caliconodewindowsdaemonsetpodtemplatespec)_ | (Optional) Template describes the calico-node-windows DaemonSet pod that will be created. | + + +### CalicoWebhooksDeployment + + + +CalicoWebhooksDeployment is the configuration for the calico-webhooks Deployment. + +_Appears in:_ +- [APIServerSpec](#apiserverspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CalicoWebhooksDeploymentSpec](#calicowebhooksdeploymentspec)_ | (Optional) Spec is the specification of the calico-webhooks Deployment. | + + +### CalicoWebhooksDeploymentContainer + + + +CalicoWebhooksDeploymentContainer is a calico-webhooks Deployment container. + +_Appears in:_ +- [CalicoWebhooksDeploymentPodSpec](#calicowebhooksdeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the calico-webhooks Deployment container by name.
Supported values are: calico-webhooks | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named calico-webhooks Deployment container's resources. If omitted, the calico-webhooks Deployment will use its default value for this container's resources. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `ports` _[CalicoWebhooksDeploymentContainerPort](#calicowebhooksdeploymentcontainerport) array_ | (Optional) Ports allows customization of the calico-webhooks container's ports. If specified, this overrides the default container port configuration. If omitted, the calico-webhooks Deployment will use its default port (6443). | + + +### CalicoWebhooksDeploymentContainerPort + + + +CalicoWebhooksDeploymentContainerPort defines a port override for a calico-webhooks container. + +_Appears in:_ +- [CalicoWebhooksDeploymentContainer](#calicowebhooksdeploymentcontainer) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the calico-webhooks Deployment container port by name.
Supported values are: calico-webhooks | +| `containerPort` _integer_ | Number of port to expose on the pod's IP address. This must be a valid port number, 0 < x < 65536. | + + +### CalicoWebhooksDeploymentPodSpec + + + +CalicoWebhooksDeploymentPodSpec is the calico-webhooks Deployment's PodSpec. + +_Appears in:_ +- [CalicoWebhooksDeploymentPodTemplateSpec](#calicowebhooksdeploymentpodtemplatespec) + +| Field | Description | +| --- | --- | +| `containers` _[CalicoWebhooksDeploymentContainer](#calicowebhooksdeploymentcontainer) array_ | (Optional) Containers is a list of calico-webhooks containers. If specified, this overrides the specified calico-webhooks Deployment containers. If omitted, the calico-webhooks Deployment will use its default values for its containers. | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) Affinity is a group of affinity scheduling rules for the calico-webhooks pods. If specified, this overrides any affinity that may be set on the calico-webhooks Deployment. If omitted, the calico-webhooks Deployment will use its default value for affinity.
WARNING: Please note that this field will override the default calico-webhooks Deployment affinity. | +| `nodeSelector` _object (keys:string, values:string)_ | NodeSelector is the calico-webhooks pod's scheduling constraints. If specified, each of the key/value pairs are added to the calico-webhooks Deployment nodeSelector provided the key does not already exist in the object's nodeSelector. If used in conjunction with ControlPlaneNodeSelector, that nodeSelector is set on the calico-webhooks Deployment and each of this field's key/value pairs are added to the calico-webhooks Deployment nodeSelector provided the key does not already exist in the object's nodeSelector. If omitted, the calico-webhooks Deployment will use its default value for nodeSelector.
WARNING: Please note that this field will modify the default calico-webhooks Deployment nodeSelector. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) Tolerations is the calico-webhooks pod's tolerations. If specified, this overrides any tolerations that may be set on the calico-webhooks Deployment. If omitted, the calico-webhooks Deployment will use its default value for tolerations.
WARNING: Please note that this field will override the default calico-webhooks Deployment tolerations. | +| `hostNetwork` _boolean_ | (Optional) HostNetwork forces the webhook pod to use the host's network namespace. When true, the webhook pod will run with hostNetwork=true and DNSPolicy=ClusterFirstWithHostNet. When nil or omitted, the operator auto-detects whether host networking is required (e.g., for EKS/TKG with Calico CNI). | + + +### CalicoWebhooksDeploymentPodTemplateSpec + + + +CalicoWebhooksDeploymentPodTemplateSpec is the calico-webhooks Deployment's PodTemplateSpec + +_Appears in:_ +- [CalicoWebhooksDeploymentSpec](#calicowebhooksdeploymentspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CalicoWebhooksDeploymentPodSpec](#calicowebhooksdeploymentpodspec)_ | (Optional) Spec is the calico-webhooks Deployment's PodSpec. | + + +### CalicoWebhooksDeploymentSpec + + + +CalicoWebhooksDeploymentSpec defines configuration for the calico-webhooks Deployment. + +_Appears in:_ +- [CalicoWebhooksDeployment](#calicowebhooksdeployment) + +| Field | Description | +| --- | --- | +| `minReadySeconds` _integer_ | (Optional) MinReadySeconds is the minimum number of seconds for which a newly created Deployment pod should be ready without any of its container crashing, for it to be considered available. If specified, this overrides any minReadySeconds value that may be set on the calico-webhooks Deployment. If omitted, the calico-webhooks Deployment will use its default value for minReadySeconds. | +| `template` _[CalicoWebhooksDeploymentPodTemplateSpec](#calicowebhooksdeploymentpodtemplatespec)_ | (Optional) Template describes the calico-webhooks Deployment pod that will be created. | + + +### CalicoWindowsUpgradeDaemonSet + + + +Deprecated. The CalicoWindowsUpgradeDaemonSet is deprecated and will be removed from the API in the future. +CalicoWindowsUpgradeDaemonSet is the configuration for the calico-windows-upgrade DaemonSet. + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CalicoWindowsUpgradeDaemonSetSpec](#calicowindowsupgradedaemonsetspec)_ | (Optional) Spec is the specification of the calico-windows-upgrade DaemonSet. | + + +### CalicoWindowsUpgradeDaemonSetContainer + + + +CalicoWindowsUpgradeDaemonSetContainer is a calico-windows-upgrade DaemonSet container. + +_Appears in:_ +- [CalicoWindowsUpgradeDaemonSetPodSpec](#calicowindowsupgradedaemonsetpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the calico-windows-upgrade DaemonSet container by name. | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named calico-windows-upgrade DaemonSet container's resources. If omitted, the calico-windows-upgrade DaemonSet will use its default value for this container's resources. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### CalicoWindowsUpgradeDaemonSetPodSpec + + + +CalicoWindowsUpgradeDaemonSetPodSpec is the calico-windows-upgrade DaemonSet's PodSpec. + +_Appears in:_ +- [CalicoWindowsUpgradeDaemonSetPodTemplateSpec](#calicowindowsupgradedaemonsetpodtemplatespec) + +| Field | Description | +| --- | --- | +| `containers` _[CalicoWindowsUpgradeDaemonSetContainer](#calicowindowsupgradedaemonsetcontainer) array_ | (Optional) Containers is a list of calico-windows-upgrade containers. If specified, this overrides the specified calico-windows-upgrade DaemonSet containers. If omitted, the calico-windows-upgrade DaemonSet will use its default values for its containers. | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) Affinity is a group of affinity scheduling rules for the calico-windows-upgrade pods. If specified, this overrides any affinity that may be set on the calico-windows-upgrade DaemonSet. If omitted, the calico-windows-upgrade DaemonSet will use its default value for affinity.
WARNING: Please note that this field will override the default calico-windows-upgrade DaemonSet affinity. | +| `nodeSelector` _object (keys:string, values:string)_ | (Optional) NodeSelector is the calico-windows-upgrade pod's scheduling constraints. If specified, each of the key/value pairs are added to the calico-windows-upgrade DaemonSet nodeSelector provided the key does not already exist in the object's nodeSelector. If omitted, the calico-windows-upgrade DaemonSet will use its default value for nodeSelector.
WARNING: Please note that this field will modify the default calico-windows-upgrade DaemonSet nodeSelector. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) Tolerations is the calico-windows-upgrade pod's tolerations. If specified, this overrides any tolerations that may be set on the calico-windows-upgrade DaemonSet. If omitted, the calico-windows-upgrade DaemonSet will use its default value for tolerations.
WARNING: Please note that this field will override the default calico-windows-upgrade DaemonSet tolerations. | + + +### CalicoWindowsUpgradeDaemonSetPodTemplateSpec + + + +CalicoWindowsUpgradeDaemonSetPodTemplateSpec is the calico-windows-upgrade DaemonSet's PodTemplateSpec + +_Appears in:_ +- [CalicoWindowsUpgradeDaemonSetSpec](#calicowindowsupgradedaemonsetspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[CalicoWindowsUpgradeDaemonSetPodSpec](#calicowindowsupgradedaemonsetpodspec)_ | (Optional) Spec is the calico-windows-upgrade DaemonSet's PodSpec. | + + +### CalicoWindowsUpgradeDaemonSetSpec + + + +CalicoWindowsUpgradeDaemonSetSpec defines configuration for the calico-windows-upgrade DaemonSet. + +_Appears in:_ +- [CalicoWindowsUpgradeDaemonSet](#calicowindowsupgradedaemonset) + +| Field | Description | +| --- | --- | +| `minReadySeconds` _integer_ | (Optional) MinReadySeconds is the minimum number of seconds for which a newly created Deployment pod should be ready without any of its container crashing, for it to be considered available. If specified, this overrides any minReadySeconds value that may be set on the calico-windows-upgrade DaemonSet. If omitted, the calico-windows-upgrade DaemonSet will use its default value for minReadySeconds. | +| `template` _[CalicoWindowsUpgradeDaemonSetPodTemplateSpec](#calicowindowsupgradedaemonsetpodtemplatespec)_ | (Optional) Template describes the calico-windows-upgrade DaemonSet pod that will be created. | + + +### CertificateManagement + + + +CertificateManagement configures pods to submit a CertificateSigningRequest to the certificates.k8s.io/v1beta1 API in order +to obtain TLS certificates. This feature requires that you bring your own CSR signing and approval process, otherwise +pods will be stuck during initialization. + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `caCert` _integer array_ | Certificate of the authority that signs the CertificateSigningRequests in PEM format. | +| `signerName` _string_ | When a CSR is issued to the certificates.k8s.io API, the signerName is added to the request in order to accommodate for clusters with multiple signers. Must be formatted as: `/`. | +| `keyAlgorithm` _string_ | (Optional) Specify the algorithm used by pods to generate a key pair that is associated with the X.509 certificate request.
Default: RSAWithSize2048 | +| `signatureAlgorithm` _string_ | (Optional) Specify the algorithm used for the signature of the X.509 certificate request.
Default: SHA256WithRSA | + + +### ClusterRoutingMode + +_Underlying type:_ _string_ + +ClusterRoutingMode describes the mode of cluster routing. + +_Validation:_ +- Enum: [BIRD Felix] + + +_Appears in:_ +- [CalicoNetworkSpec](#caliconetworkspec) + +| Value | Description | +| --- | --- | +| `BIRD` | | +| `Felix` | | + + +### CollectProcessPathOption + +_Underlying type:_ _string_ + + + +_Appears in:_ +- [LogCollectorSpec](#logcollectorspec) + +| Value | Description | +| --- | --- | +| `Enabled` | | +| `Disabled` | | + + +### CommonPrometheusFields + + + + + +_Appears in:_ +- [PrometheusSpec](#prometheusspec) + +| Field | Description | +| --- | --- | +| `containers` _[PrometheusContainer](#prometheuscontainer) array_ | (Optional) Containers is a list of Prometheus containers. If specified, this overrides the specified Prometheus Deployment containers. If omitted, the Prometheus Deployment will use its default values for its containers. | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | Define resources requests and limits for single Pods. | + + +### ComponentName + +_Underlying type:_ _string_ + +ComponentName represents a single component. + +One of: Node, Typha, KubeControllers + +_Appears in:_ +- [ComponentResource](#componentresource) + +| Value | Description | +| --- | --- | +| `Node` | | +| `NodeWindows` | | +| `FelixWindows` | | +| `ConfdWindows` | | +| `Typha` | | +| `KubeControllers` | | + + +### ComponentResource + + + +Deprecated. Please use component resource config fields in Installation.Spec instead. +The ComponentResource struct associates a ResourceRequirements with a component by name + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `componentName` _[ComponentName](#componentname)_ | ComponentName is an enum which identifies the component | +| `resourceRequirements` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | ResourceRequirements allows customization of limits and requests for compute resources such as cpu and memory. | + + +### ConditionStatus + +_Underlying type:_ _string_ + +ConditionStatus represents the status of a particular condition. A condition may be one of: True, False, Unknown. + +_Appears in:_ +- [TigeraStatusCondition](#tigerastatuscondition) + +| Value | Description | +| --- | --- | +| `True` | | +| `False` | | +| `Unknown` | | + + +### ContainerIPForwardingType + +_Underlying type:_ _string_ + +ContainerIPForwardingType specifies whether the CNI config for container ip forwarding is enabled. + +_Appears in:_ +- [CalicoNetworkSpec](#caliconetworkspec) + +| Value | Description | +| --- | --- | +| `Enabled` | | +| `Disabled` | | + + +### DPIDaemonsetInitContainer + + + + + +_Appears in:_ +- [DPIDaemonsetTemplateSpec](#dpidaemonsettemplatespec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum that identifies the init container by its name. | +| `image` _string_ | Image name for the init container | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the init container's resources. If omitted, the default values will be used for the init container's resources. | + + +### DPIDaemonsetSpec + + + + + +_Appears in:_ +- [DeepPacketInspectionDaemonset](#deeppacketinspectiondaemonset) + +| Field | Description | +| --- | --- | +| `template` _[DPIDaemonsetTemplate](#dpidaemonsettemplate)_ | (Optional) Template specifies DPI Daemonset Template | + + +### DPIDaemonsetTemplate + + + + + +_Appears in:_ +- [DPIDaemonsetSpec](#dpidaemonsetspec) + +| Field | Description | +| --- | --- | +| `spec` _[DPIDaemonsetTemplateSpec](#dpidaemonsettemplatespec)_ | (Optional) Spec specifies DPI Daemonset Template Spec | + + +### DPIDaemonsetTemplateSpec + + + + + +_Appears in:_ +- [DPIDaemonsetTemplate](#dpidaemonsettemplate) + +| Field | Description | +| --- | --- | +| `initContainers` _[DPIDaemonsetInitContainer](#dpidaemonsetinitcontainer) array_ | List of DPI Daemonset Init containers definitions | + + +### DeepPacketInspectionDaemonset + + + + + +_Appears in:_ +- [IntrusionDetectionSpec](#intrusiondetectionspec) + +| Field | Description | +| --- | --- | +| `spec` _[DPIDaemonsetSpec](#dpidaemonsetspec)_ | (Optional) DPIDaemonsetSpec configures the DPI Daemonset | + + +### EGWDeploymentContainer + + + +EGWDeploymentContainer is a Egress Gateway Deployment container. + +_Appears in:_ +- [EgressGatewayDeploymentPodSpec](#egressgatewaydeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the EGW Deployment container by name.
Supported values are: egress-gateway | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named EGW Deployment container's resources. If omitted, the EGW Deployment will use its default value for this container's resources. If used in conjunction with the deprecated ComponentResources, then this value takes precedence. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### EGWDeploymentInitContainer + + + +EGWDeploymentInitContainer is a Egress Gateway Deployment init container. + +_Appears in:_ +- [EgressGatewayDeploymentPodSpec](#egressgatewaydeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the EGW Deployment init container by name.
Supported values are: egress-gateway-init | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named EGW Deployment init container's resources. If omitted, the EGW Deployment will use its default value for this init container's resources. If used in conjunction with the deprecated ComponentResources, then this value takes precedence. | + + +### EKSLogForwarderDeployment + + + +EKSLogForwarderDeployment is the configuration for the EKSLogForwarder Deployment. + +_Appears in:_ +- [LogCollectorSpec](#logcollectorspec) + +| Field | Description | +| --- | --- | +| `spec` _[EKSLogForwarderDeploymentSpec](#ekslogforwarderdeploymentspec)_ | (Optional) Spec is the specification of the EKSLogForwarder Deployment. | + + +### EKSLogForwarderDeploymentContainer + + + +EKSLogForwarderDeploymentContainer is a EKSLogForwarder Deployment container. + +_Appears in:_ +- [EKSLogForwarderDeploymentPodSpec](#ekslogforwarderdeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the EKSLogForwarder Deployment container by name.
Supported values are: eks-log-forwarder | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named EKSLogForwarder Deployment container's resources. If omitted, the EKSLogForwarder Deployment will use its default value for this container's resources. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### EKSLogForwarderDeploymentInitContainer + + + +EKSLogForwarderDeploymentInitContainer is a EKSLogForwarder Deployment init container. + +_Appears in:_ +- [EKSLogForwarderDeploymentPodSpec](#ekslogforwarderdeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the EKSLogForwarder Deployment init container by name.
Supported values are: eks-log-forwarder-startup | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named EKSLogForwarder Deployment init container's resources. If omitted, the EKSLogForwarder Deployment will use its default value for this init container's resources. | + + +### EKSLogForwarderDeploymentPodSpec + + + +EKSLogForwarderDeploymentPodSpec is the EKSLogForwarder Deployment's PodSpec. + +_Appears in:_ +- [EKSLogForwarderDeploymentPodTemplateSpec](#ekslogforwarderdeploymentpodtemplatespec) + +| Field | Description | +| --- | --- | +| `initContainers` _[EKSLogForwarderDeploymentInitContainer](#ekslogforwarderdeploymentinitcontainer) array_ | (Optional) InitContainers is a list of EKSLogForwarder init containers. If specified, this overrides the specified EKSLogForwarder Deployment init containers. If omitted, the EKSLogForwarder Deployment will use its default values for its init containers. | +| `containers` _[EKSLogForwarderDeploymentContainer](#ekslogforwarderdeploymentcontainer) array_ | (Optional) Containers is a list of EKSLogForwarder containers. If specified, this overrides the specified EKSLogForwarder Deployment containers. If omitted, the EKSLogForwarder Deployment will use its default values for its containers. | + + +### EKSLogForwarderDeploymentPodTemplateSpec + + + +EKSLogForwarderDeploymentPodTemplateSpec is the EKSLogForwarder Deployment's PodTemplateSpec + +_Appears in:_ +- [EKSLogForwarderDeploymentSpec](#ekslogforwarderdeploymentspec) + +| Field | Description | +| --- | --- | +| `spec` _[EKSLogForwarderDeploymentPodSpec](#ekslogforwarderdeploymentpodspec)_ | (Optional) Spec is the EKSLogForwarder Deployment's PodSpec. | + + +### EKSLogForwarderDeploymentSpec + + + +EKSLogForwarderDeploymentSpec defines configuration for the EKSLogForwarder Deployment. + +_Appears in:_ +- [EKSLogForwarderDeployment](#ekslogforwarderdeployment) + +| Field | Description | +| --- | --- | +| `template` _[EKSLogForwarderDeploymentPodTemplateSpec](#ekslogforwarderdeploymentpodtemplatespec)_ | (Optional) Template describes the EKSLogForwarder Deployment pod that will be created. | + + +### EgressGateway + + + +EgressGateway is the Schema for the egressgateways API + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `EgressGateway` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[EgressGatewaySpec](#egressgatewayspec)_ | | +| `status` _[EgressGatewayStatus](#egressgatewaystatus)_ | | + + +### EgressGatewayDeploymentPodSpec + + + +EgressGatewayDeploymentPodSpec is the Egress Gateway Deployment's PodSpec. + +_Appears in:_ +- [EgressGatewayDeploymentPodTemplateSpec](#egressgatewaydeploymentpodtemplatespec) + +| Field | Description | +| --- | --- | +| `initContainers` _[EGWDeploymentInitContainer](#egwdeploymentinitcontainer) array_ | (Optional) InitContainers is a list of EGW init containers. If specified, this overrides the specified EGW Deployment init containers. If omitted, the EGW Deployment will use its default values for its init containers. | +| `containers` _[EGWDeploymentContainer](#egwdeploymentcontainer) array_ | (Optional) Containers is a list of EGW containers. If specified, this overrides the specified EGW Deployment containers. If omitted, the EGW Deployment will use its default values for its containers. | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) Affinity is a group of affinity scheduling rules for the EGW pods. | +| `nodeSelector` _object (keys:string, values:string)_ | (Optional) NodeSelector gives more control over the nodes where the Egress Gateway pods will run on. | +| `terminationGracePeriodSeconds` _integer_ | (Optional) TerminationGracePeriodSeconds defines the termination grace period of the Egress Gateway pods in seconds. | +| `topologySpreadConstraints` _[TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#topologyspreadconstraint-v1-core) array_ | (Optional) TopologySpreadConstraints defines how the Egress Gateway pods should be spread across different AZs. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) Tolerations is the egress gateway pod's tolerations. If specified, this overrides any tolerations that may be set on the EGW Deployment. If omitted, the EGW Deployment will use its default value for tolerations. | +| `priorityClassName` _string_ | (Optional) PriorityClassName allows to specify a PriorityClass resource to be used. | + + +### EgressGatewayDeploymentPodTemplateSpec + + + +EgressGatewayDeploymentPodTemplateSpec is the EGW Deployment's PodTemplateSpec + +_Appears in:_ +- [EgressGatewaySpec](#egressgatewayspec) + +| Field | Description | +| --- | --- | +| `metadata` _[EgressGatewayMetadata](#egressgatewaymetadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[EgressGatewayDeploymentPodSpec](#egressgatewaydeploymentpodspec)_ | (Optional) Spec is the EGW Deployment's PodSpec. | + + +### EgressGatewayFailureDetection + + + +EgressGatewayFailureDetection defines the fields the needed for determining Egress Gateway +readiness. + +_Appears in:_ +- [EgressGatewaySpec](#egressgatewayspec) + +| Field | Description | +| --- | --- | +| `healthTimeoutDataStoreSeconds` _integer_ | (Optional) HealthTimeoutDataStoreSeconds defines how long Egress Gateway can fail to connect to the datastore before reporting not ready. This value must be greater than 0.
Default: 90 | +| `icmpProbe` _[ICMPProbe](#icmpprobe)_ | (Optional) ICMPProbe define outgoing ICMP probes that Egress Gateway will use to verify its upstream connection. Egress Gateway will report not ready if all fail. Timeout must be greater than interval. | +| `httpProbe` _[HTTPProbe](#httpprobe)_ | (Optional) HTTPProbe define outgoing HTTP probes that Egress Gateway will use to verify its upsteam connection. Egress Gateway will report not ready if all fail. Timeout must be greater than interval. | + + +### EgressGatewayIPPool + + + + + +_Appears in:_ +- [EgressGatewaySpec](#egressgatewayspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | (Optional) Name is the name of the IPPool that the Egress Gateways can use. | +| `cidr` _string_ | (Optional) CIDR is the IPPool CIDR that the Egress Gateways can use. | + + +### EgressGatewayMetadata + + + +EgressGatewayMetadata contains the standard Kubernetes labels and annotations fields. + +_Appears in:_ +- [EgressGatewayDeploymentPodTemplateSpec](#egressgatewaydeploymentpodtemplatespec) + +| Field | Description | +| --- | --- | +| `labels` _object (keys:string, values:string)_ | (Optional) Labels is a map of string keys and values that may match replica set and service selectors. Each of these key/value pairs are added to the object's labels provided the key does not already exist in the object's labels. If not specified will default to projectcalico.org/egw:[name], where [name] is the name of the Egress Gateway resource. | +| `annotations` _object (keys:string, values:string)_ | (Optional) Annotations is a map of arbitrary non-identifying metadata. Each of these key/value pairs are added to the object's annotations provided the key does not already exist in the object's annotations. | + + +### EgressGatewaySpec + + + +EgressGatewaySpec defines the desired state of EgressGateway + +_Appears in:_ +- [EgressGateway](#egressgateway) + +| Field | Description | +| --- | --- | +| `replicas` _integer_ | (Optional) Replicas defines how many instances of the Egress Gateway pod will run. | +| `ipPools` _[EgressGatewayIPPool](#egressgatewayippool) array_ | IPPools defines the IP Pools that the Egress Gateway pods should be using. Either name or CIDR must be specified. IPPools must match existing IPPools. | +| `externalNetworks` _string array_ | (Optional) ExternalNetworks defines the external network names this Egress Gateway is associated with. ExternalNetworks must match existing external networks. | +| `logSeverity` _[LogSeverity](#logseverity)_ | (Optional) LogSeverity defines the logging level of the Egress Gateway. | +| `template` _[EgressGatewayDeploymentPodTemplateSpec](#egressgatewaydeploymentpodtemplatespec)_ | (Optional) Template describes the EGW Deployment pod that will be created. | +| `egressGatewayFailureDetection` _[EgressGatewayFailureDetection](#egressgatewayfailuredetection)_ | (Optional) EgressGatewayFailureDetection is used to configure how Egress Gateway determines readiness. If both ICMP, HTTP probes are defined, one ICMP probe and one HTTP probe should succeed for Egress Gateways to become ready. Otherwise one of ICMP or HTTP probe should succeed for Egress gateways to become ready if configured. | +| `aws` _[AWSEgressGateway](#awsegressgateway)_ | (Optional) AWS defines the additional configuration options for Egress Gateways on AWS. | + + +### EgressGatewayStatus + + + +EgressGatewayStatus defines the observed state of EgressGateway + +_Appears in:_ +- [EgressGateway](#egressgateway) + +| Field | Description | +| --- | --- | +| `state` _string_ | State provides user-readable status. | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#condition-v1-meta) array_ | (Optional) Conditions represents the latest observed set of conditions for the component. A component may be one or more of Ready, Progressing, Degraded or other customer types. | + + +### EksCloudwatchLogsSpec + + + +EksConfigSpec defines configuration for fetching EKS audit logs. + +_Appears in:_ +- [AdditionalLogSourceSpec](#additionallogsourcespec) + +| Field | Description | +| --- | --- | +| `region` _string_ | AWS Region EKS cluster is hosted in. | +| `groupName` _string_ | Cloudwatch log-group name containing EKS audit logs. | +| `streamPrefix` _string_ | (Optional) Prefix of Cloudwatch log stream containing EKS audit logs in the log-group.
Default: kube-apiserver-audit- | +| `fetchInterval` _integer_ | (Optional) Cloudwatch audit logs fetching interval in seconds.
Default: 60 | + + +### EncapsulationType + +_Underlying type:_ _string_ + +EncapsulationType is the type of encapsulation to use on an IP pool. + +One of: IPIP, VXLAN, IPIPCrossSubnet, VXLANCrossSubnet, None + +_Appears in:_ +- [IPPool](#ippool) + +| Value | Description | +| --- | --- | +| `IPIPCrossSubnet` | | +| `IPIP` | | +| `VXLAN` | | +| `VXLANCrossSubnet` | | +| `None` | | + + +### EncryptionOption + +_Underlying type:_ _string_ + +EncryptionOption specifies the traffic encryption mode when connecting to a Syslog server. + +One of: None, TLS + +_Appears in:_ +- [SyslogStoreSpec](#syslogstorespec) + +| Value | Description | +| --- | --- | +| `None` | | +| `TLS` | | + + +### Endpoint + + + +Endpoint contains a subset of relevant fields from the Prometheus Endpoint struct. + +_Appears in:_ +- [ServiceMonitor](#servicemonitor) + +| Field | Description | +| --- | --- | +| `params` _object (keys:string, values:string array)_ | Optional HTTP URL parameters
Default: scrape all metrics. | +| `bearerTokenSecret` _[SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#secretkeyselector-v1-core)_ | Secret to mount to read bearer token for scraping targets. Recommended: when unset, the operator will create a Secret, a ClusterRole and a ClusterRoleBinding. | +| `interval` _[Duration](#duration)_ | Interval at which metrics should be scraped. If not specified Prometheus' global scrape interval is used. | +| `scrapeTimeout` _[Duration](#duration)_ | Timeout after which the scrape is ended. If not specified, the Prometheus global scrape timeout is used unless it is less than `Interval` in which the latter is used. | +| `honorLabels` _boolean_ | HonorLabels chooses the metric's labels on collisions with target labels. | +| `honorTimestamps` _boolean_ | HonorTimestamps controls whether Prometheus respects the timestamps present in scraped data. | +| `metricRelabelings` _RelabelConfig array_ | MetricRelabelConfigs to apply to samples before ingestion. | +| `relabelings` _RelabelConfig array_ | RelabelConfigs to apply to samples before scraping. Prometheus Operator automatically adds relabelings for a few standard Kubernetes fields. The original scrape job's name is available via the `__tmp_prometheus_job_name` label. More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config | + + +### EnvoySettings + + + + + +_Appears in:_ +- [ApplicationLayerSpec](#applicationlayerspec) + +| Field | Description | +| --- | --- | +| `xffNumTrustedHops` _integer_ | (Optional) The number of additional ingress proxy hops from the right side of the x-forwarded-for HTTP header to trust when determining the origin client’s IP address. 0 is permitted, but >=1 is the typical setting. | +| `useRemoteAddress` _boolean_ | (Optional) If set to true, the Envoy connection manager will use the real remote address of the client connection when determining internal versus external origin and manipulating various headers. | + + +### ExternalPrometheus + + + + + +_Appears in:_ +- [MonitorSpec](#monitorspec) + +| Field | Description | +| --- | --- | +| `serviceMonitor` _[ServiceMonitor](#servicemonitor)_ | (Optional) ServiceMonitor when specified, the operator will create a ServiceMonitor object in the namespace. It is recommended that you configure labels if you want your prometheus instance to pick up the configuration automatically. The operator will configure 1 endpoint by default: - Params to scrape all metrics available in Calico Enterprise. - BearerTokenSecret (If not overridden, the operator will also create corresponding RBAC that allows authz to the metrics.) - TLSConfig, containing the caFile and serverName. | +| `namespace` _string_ | Namespace is the namespace where the operator will create resources for your Prometheus instance. The namespace must be created before the operator will create Prometheus resources. | + + +### FIPSMode + +_Underlying type:_ _string_ + + + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Value | Description | +| --- | --- | +| `Enabled` | | +| `Disabled` | | + + +### FluentdDaemonSet + + + +FluentdDaemonSet is the configuration for the Fluentd DaemonSet. + +_Appears in:_ +- [LogCollectorSpec](#logcollectorspec) + +| Field | Description | +| --- | --- | +| `spec` _[FluentdDaemonSetSpec](#fluentddaemonsetspec)_ | (Optional) Spec is the specification of the Fluentd DaemonSet. | + + +### FluentdDaemonSetContainer + + + +FluentdDaemonSetContainer is a Fluentd DaemonSet container. + +_Appears in:_ +- [FluentdDaemonSetPodSpec](#fluentddaemonsetpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the Fluentd DaemonSet container by name.
Supported values are: fluentd | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named Fluentd DaemonSet container's resources. If omitted, the Fluentd DaemonSet will use its default value for this container's resources. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### FluentdDaemonSetInitContainer + + + +FluentdDaemonSetInitContainer is a Fluentd DaemonSet init container. + +_Appears in:_ +- [FluentdDaemonSetPodSpec](#fluentddaemonsetpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the Fluentd DaemonSet init container by name.
Supported values are: tigera-fluentd-prometheus-tls-key-cert-provisioner | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named Fluentd DaemonSet init container's resources. If omitted, the Fluentd DaemonSet will use its default value for this init container's resources. | + + +### FluentdDaemonSetPodSpec + + + +FluentdDaemonSetPodSpec is the Fluentd DaemonSet's PodSpec. + +_Appears in:_ +- [FluentdDaemonSetPodTemplateSpec](#fluentddaemonsetpodtemplatespec) + +| Field | Description | +| --- | --- | +| `initContainers` _[FluentdDaemonSetInitContainer](#fluentddaemonsetinitcontainer) array_ | (Optional) InitContainers is a list of Fluentd DaemonSet init containers. If specified, this overrides the specified Fluentd DaemonSet init containers. If omitted, the Fluentd DaemonSet will use its default values for its init containers. | +| `containers` _[FluentdDaemonSetContainer](#fluentddaemonsetcontainer) array_ | (Optional) Containers is a list of Fluentd DaemonSet containers. If specified, this overrides the specified Fluentd DaemonSet containers. If omitted, the Fluentd DaemonSet will use its default values for its containers. | + + +### FluentdDaemonSetPodTemplateSpec + + + +FluentdDaemonSetPodTemplateSpec is the Fluentd DaemonSet's PodTemplateSpec + +_Appears in:_ +- [FluentdDaemonSetSpec](#fluentddaemonsetspec) + +| Field | Description | +| --- | --- | +| `spec` _[FluentdDaemonSetPodSpec](#fluentddaemonsetpodspec)_ | (Optional) Spec is the Fluentd DaemonSet's PodSpec. | + + +### FluentdDaemonSetSpec + + + +FluentdDaemonSetSpec defines configuration for the Fluentd DaemonSet. + +_Appears in:_ +- [FluentdDaemonSet](#fluentddaemonset) + +| Field | Description | +| --- | --- | +| `template` _[FluentdDaemonSetPodTemplateSpec](#fluentddaemonsetpodtemplatespec)_ | (Optional) Template describes the Fluentd DaemonSet pod that will be created. | + + +### GatewayAPI + + + + + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `GatewayAPI` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[GatewayAPISpec](#gatewayapispec)_ | | + + +### GatewayAPIExtensions + + + +GatewayAPIExtensions enables and configures Tigera-built Gateway API add-ons. + +_Appears in:_ +- [GatewayAPISpec](#gatewayapispec) + +| Field | Description | +| --- | --- | +| `waf` _[WAFExtensionSpec](#wafextensionspec)_ | (Optional) WAF enables and configures the Tigera Web Application Firewall (Coraza WASM when WAF.State is nil, and when WAF.State is "Disabled", the operator does not render the WAF env vars or RBAC on calico-kube-controllers. Set WAF.State = "Enabled" to turn the feature on. See design `tigera/designs#25` (PMREQ-384) for the full surface. | + + +### GatewayAPISpec + + + +GatewayAPISpec has fields that can be used to customize our GatewayAPI support. + +_Appears in:_ +- [GatewayAPI](#gatewayapi) + +| Field | Description | +| --- | --- | +| `envoyGatewayConfigRef` _[NamespacedName](#namespacedname)_ | (Optional) Reference to a custom EnvoyGateway YAML to use as the base EnvoyGateway configuration for the gateway controller. When specified, must identify a ConfigMap resource with an "envoy-gateway.yaml" key whose value is the desired EnvoyGateway YAML (i.e. following the same pattern as the default `envoy-gateway-config` ConfigMap). When not specified, the Tigera operator uses the `envoy-gateway-config` from the Envoy Gateway helm chart as its base. Starting from that base, the Tigera operator copies and modifies the EnvoyGateway resource as follows: 1. If not already specified, it sets the ControllerName to "gateway.envoyproxy.io/gatewayclass-controller". 2. It configures the `tigera/envoy-gateway` and `tigera/envoy-ratelimit` images that will be used (according to the current Calico version, private registry and image set settings) and any pull secrets that are needed to pull those images. 3. It enables use of the Backend API. The resulting EnvoyGateway is provisioned as the `envoy-gateway-config` ConfigMap (which the gateway controller then uses as its config). | +| `gatewayClasses` _[GatewayClassSpec](#gatewayclassspec) array_ | (Optional) Configures the GatewayClasses that will be available; please see GatewayClassSpec for more detail. If GatewayClasses is nil, the Tigera operator defaults to provisioning a single GatewayClass named "tigera-gateway-class", without any of the detailed customizations that are allowed within GatewayClassSpec. | +| `gatewayControllerDeployment` _[GatewayControllerDeployment](#gatewaycontrollerdeployment)_ | (Optional) Allows customization of the gateway controller deployment. | +| `gatewayCertgenJob` _[GatewayCertgenJob](#gatewaycertgenjob)_ | (Optional) Allows customization of the gateway certgen job. | +| `crdManagement` _[CRDManagement](#crdmanagement)_ | (Optional) Configures how to manage and update Gateway API CRDs. The default behaviour - which is used when this field is not set, or is set to "PreferExisting" - is that the Tigera operator will create the Gateway API CRDs if they do not already exist, but will not overwrite any existing Gateway API CRDs. This setting may be preferable if the customer is using other implementations of the Gateway API concurrently with the Gateway API support in Calico Enterprise. It is then the customer's responsibility to ensure that CRDs are installed that meet the needs of all the Gateway API implementations in their cluster. Alternatively, if this field is set to "Reconcile", the Tigera operator will keep the cluster's Gateway API CRDs aligned with those that it would install on a cluster that does not yet have any version of those CRDs. | +| `extensions` _[GatewayAPIExtensions](#gatewayapiextensions)_ | (Optional) Extensions enables and configures Tigera-built add-ons that sit on top of the Gateway API data plane. Each add-on is opt-in: an unset Extensions, an unset add-on field, and an empty add-on object all leave the add-on disabled. | + + +### GatewayCertgenJob + + + +GatewayCertgenJob allows customization of the gateway certgen job. + +_Appears in:_ +- [GatewayAPISpec](#gatewayapispec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[GatewayCertgenJobSpec](#gatewaycertgenjobspec)_ | (Optional) | + + +### GatewayCertgenJobContainer + + + +GatewayCertgenJobContainer allows customization of the gateway certgen job's resource +requirements. + +_Appears in:_ +- [GatewayCertgenJobPodSpec](#gatewaycertgenjobpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) If non-nil, Resources sets the ResourceRequirements of the job's "envoy-gateway-certgen" container. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### GatewayCertgenJobPodSpec + + + +GatewayCertgenJobPodSpec allows customization of the gateway certgen job's pod spec. + +_Appears in:_ +- [GatewayCertgenJobPodTemplate](#gatewaycertgenjobpodtemplate) + +| Field | Description | +| --- | --- | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) If non-nil, Affinity sets the affinity field of the job's pod template. | +| `containers` _[GatewayCertgenJobContainer](#gatewaycertgenjobcontainer) array_ | (Optional) | +| `nodeSelector` _object (keys:string, values:string)_ | (Optional) If non-nil, NodeSelector sets the node selector for where job pods may be scheduled. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) If non-nil, Tolerations sets the tolerations field of the job's pod template. | + + +### GatewayCertgenJobPodTemplate + + + +GatewayCertgenJobPodTemplate allows customization of the gateway certgen job's pod template. + +_Appears in:_ +- [GatewayCertgenJobSpec](#gatewaycertgenjobspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[GatewayCertgenJobPodSpec](#gatewaycertgenjobpodspec)_ | (Optional) | + + +### GatewayCertgenJobSpec + + + +GatewayCertgenJobSpec allows customization of the gateway certgen job spec. + +_Appears in:_ +- [GatewayCertgenJob](#gatewaycertgenjob) + +| Field | Description | +| --- | --- | +| `template` _[GatewayCertgenJobPodTemplate](#gatewaycertgenjobpodtemplate)_ | (Optional) | + + +### GatewayClassSpec + + + + + +_Appears in:_ +- [GatewayAPISpec](#gatewayapispec) + +| Field | Description | +| --- | --- | +| `name` _string_ | The name of this GatewayClass. | +| `envoyProxyRef` _[NamespacedName](#namespacedname)_ | (Optional) Reference to a custom EnvoyProxy resource to use as the base EnvoyProxy configuration for this GatewayClass. When specified, must identify an EnvoyProxy resource. When not specified, the Tigera operator uses an empty EnvoyProxy resource as its base. Starting from that base, the Tigera operator copies and modifies the EnvoyProxy resource as follows, in the order described: 1. It configures the `tigera/envoy-proxy` image that will be used (according to the current Calico version, private registry and image set settings) and any pull secrets that are needed to pull that image. 2. It applies customizations as specified by the following `GatewayKind`, `GatewayDeployment`, `GatewayDaemonSet` and `GatewayService` fields. The resulting EnvoyProxy is provisioned in the `tigera-gateway` namespace, together with a GatewayClass that references it. If a custom EnvoyProxy resource is specified and uses `EnvoyDaemonSet` instead of the default `EnvoyDeployment`, deployment-related customizations will be applied within `EnvoyDaemonSet` instead of within `EnvoyDeployment`. | +| `gatewayKind` _[GatewayKind](#gatewaykind)_ | (Optional) Specifies whether Gateways in this class are deployed as Deployments (default) or as DaemonSets. It is an error for GatewayKind to specify a choice that is incompatible with the custom EnvoyProxy, when EnvoyProxyRef is also specified. | +| `gatewayDeployment` _[GatewayDeployment](#gatewaydeployment)_ | (Optional) Allows customization of Gateways when deployed as Kubernetes Deployments, for Gateways in this GatewayClass. | +| `gatewayDaemonSet` _[GatewayDaemonSet](#gatewaydaemonset)_ | (Optional) Allows customization of Gateways when deployed as Kubernetes DaemonSets, for Gateways in this GatewayClass. | +| `gatewayService` _[GatewayService](#gatewayservice)_ | (Optional) Allows customization of gateway services, for Gateways in this GatewayClass. | + + +### GatewayControllerDeployment + + + +GatewayControllerDeployment allows customization of the gateway controller deployment. + +_Appears in:_ +- [GatewayAPISpec](#gatewayapispec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[GatewayControllerDeploymentSpec](#gatewaycontrollerdeploymentspec)_ | (Optional) | + + +### GatewayControllerDeploymentContainer + + + +GatewayControllerDeploymentContainer allows customization of the gateway controller's resource +requirements. + +_Appears in:_ +- [GatewayControllerDeploymentPodSpec](#gatewaycontrollerdeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) If non-nil, Resources sets the ResourceRequirements of the controller's "envoy-gateway" container. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### GatewayControllerDeploymentPodSpec + + + +GatewayControllerDeploymentPodSpec allows customization of the gateway controller deployment pod +spec. + +_Appears in:_ +- [GatewayControllerDeploymentPodTemplate](#gatewaycontrollerdeploymentpodtemplate) + +| Field | Description | +| --- | --- | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) If non-nil, Affinity sets the affinity field of the deployment's pod template. | +| `containers` _[GatewayControllerDeploymentContainer](#gatewaycontrollerdeploymentcontainer) array_ | (Optional) | +| `nodeSelector` _object (keys:string, values:string)_ | (Optional) If non-nil, NodeSelector sets the node selector for where deployment pods may be scheduled. | +| `topologySpreadConstraints` _[TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#topologyspreadconstraint-v1-core) array_ | (Optional) If non-nil, TopologySpreadConstraints sets the topology spread constraints of the deployment's pod template. TopologySpreadConstraints describes how a group of pods ought to spread across topology domains. Scheduler will schedule pods in a way which abides by the constraints. All topologySpreadConstraints are ANDed. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) If non-nil, Tolerations sets the tolerations field of the deployment's pod template. | + + +### GatewayControllerDeploymentPodTemplate + + + +GatewayControllerDeploymentPodTemplate allows customization of the gateway controller deployment +pod template. + +_Appears in:_ +- [GatewayControllerDeploymentSpec](#gatewaycontrollerdeploymentspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[GatewayControllerDeploymentPodSpec](#gatewaycontrollerdeploymentpodspec)_ | (Optional) | + + +### GatewayControllerDeploymentSpec + + + +GatewayControllerDeploymentSpec allows customization of the gateway controller deployment spec. + +_Appears in:_ +- [GatewayControllerDeployment](#gatewaycontrollerdeployment) + +| Field | Description | +| --- | --- | +| `replicas` _integer_ | (Optional) If non-nil, Replicas sets the number of replicas for the deployment. | +| `minReadySeconds` _integer_ | (Optional) If non-nil, MinReadySeconds sets the minReadySeconds field for the deployment. | +| `template` _[GatewayControllerDeploymentPodTemplate](#gatewaycontrollerdeploymentpodtemplate)_ | (Optional) | + + +### GatewayDaemonSet + + + +GatewayDeployment allows customization of Gateways when deployed as Kubernetes DaemonSets. + +_Appears in:_ +- [GatewayClassSpec](#gatewayclassspec) + +| Field | Description | +| --- | --- | +| `spec` _[GatewayDaemonSetSpec](#gatewaydaemonsetspec)_ | (Optional) | + + +### GatewayDaemonSetContainer + + + +GatewayDaemonSetContainer allows customization of the resource requirements of gateway +daemonsets. + +_Appears in:_ +- [GatewayDaemonSetPodSpec](#gatewaydaemonsetpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) container. | + + +### GatewayDaemonSetPodSpec + + + +GatewayDaemonSetPodSpec allows customization of the pod spec of gateway daemonsets. + +_Appears in:_ +- [GatewayDaemonSetPodTemplate](#gatewaydaemonsetpodtemplate) + +| Field | Description | +| --- | --- | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) If non-nil, Affinity sets the affinity field of the daemonset's pod template. | +| `containers` _[GatewayDaemonSetContainer](#gatewaydaemonsetcontainer) array_ | (Optional) | +| `nodeSelector` _object (keys:string, values:string)_ | (Optional) If non-nil, NodeSelector sets the node selector for where daemonset pods may be scheduled. | +| `topologySpreadConstraints` _[TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#topologyspreadconstraint-v1-core) array_ | (Optional) If non-nil, TopologySpreadConstraints sets the topology spread constraints of the daemonset's pod template. TopologySpreadConstraints describes how a group of pods ought to spread across topology domains. Scheduler will schedule pods in a way which abides by the constraints. All topologySpreadConstraints are ANDed. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) If non-nil, Tolerations sets the tolerations field of the daemonset's pod template. | + + +### GatewayDaemonSetPodTemplate + + + +GatewayDeploymentPodTemplate allows customization of the pod template of gateway daemonsets. + +_Appears in:_ +- [GatewayDaemonSetSpec](#gatewaydaemonsetspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[GatewayDaemonSetPodSpec](#gatewaydaemonsetpodspec)_ | (Optional) | + + +### GatewayDaemonSetSpec + + + +GatewayDeploymentSpec allows customization of the spec of gateway daemonsets. + +_Appears in:_ +- [GatewayDaemonSet](#gatewaydaemonset) + +| Field | Description | +| --- | --- | +| `template` _[GatewayDaemonSetPodTemplate](#gatewaydaemonsetpodtemplate)_ | (Optional) | + + +### GatewayDeployment + + + +GatewayDeployment allows customization of Gateways when deployed as Kubernetes Deployments. + +_Appears in:_ +- [GatewayClassSpec](#gatewayclassspec) + +| Field | Description | +| --- | --- | +| `spec` _[GatewayDeploymentSpec](#gatewaydeploymentspec)_ | (Optional) | + + +### GatewayDeploymentContainer + + + +GatewayDeploymentContainer allows customization of the resource requirements of gateway +deployments. + +_Appears in:_ +- [GatewayDeploymentPodSpec](#gatewaydeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) If non-nil, Resources sets the ResourceRequirements of the deployment's "envoy" container. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### GatewayDeploymentPodSpec + + + +GatewayDeploymentPodSpec allows customization of the pod spec of gateway deployments. + +_Appears in:_ +- [GatewayDeploymentPodTemplate](#gatewaydeploymentpodtemplate) + +| Field | Description | +| --- | --- | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) If non-nil, Affinity sets the affinity field of the deployment's pod template. | +| `containers` _[GatewayDeploymentContainer](#gatewaydeploymentcontainer) array_ | (Optional) | +| `nodeSelector` _object (keys:string, values:string)_ | (Optional) If non-nil, NodeSelector sets the node selector for where deployment pods may be scheduled. | +| `topologySpreadConstraints` _[TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#topologyspreadconstraint-v1-core) array_ | (Optional) If non-nil, TopologySpreadConstraints sets the topology spread constraints of the deployment's pod template. TopologySpreadConstraints describes how a group of pods ought to spread across topology domains. Scheduler will schedule pods in a way which abides by the constraints. All topologySpreadConstraints are ANDed. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) If non-nil, Tolerations sets the tolerations field of the deployment's pod template. | + + +### GatewayDeploymentPodTemplate + + + +GatewayDeploymentPodTemplate allows customization of the pod template of gateway deployments. + +_Appears in:_ +- [GatewayDeploymentSpec](#gatewaydeploymentspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[GatewayDeploymentPodSpec](#gatewaydeploymentpodspec)_ | (Optional) | + + +### GatewayDeploymentSpec + + + +GatewayDeploymentSpec allows customization of the spec of gateway deployments. + +_Appears in:_ +- [GatewayDeployment](#gatewaydeployment) + +| Field | Description | +| --- | --- | +| `replicas` _integer_ | (Optional) If non-nil, Replicas sets the number of replicas for the deployment. | +| `template` _[GatewayDeploymentPodTemplate](#gatewaydeploymentpodtemplate)_ | (Optional) | +| `strategy` _[GatewayDeploymentStrategy](#gatewaydeploymentstrategy)_ | (Optional) The deployment strategy to use to replace existing pods with new ones. | + + +### GatewayDeploymentStrategy + + + +GatewayDeploymentStrategy allows customization of the deployment strategy for gateway +deployments. + +If GatewayDeployment.Spec.Strategy is non-nil, gateway deployments are set to use a rolling +update strategy, with the parameters specified in GatewayDeployment.Spec.Strategy. + +Only RollingUpdate is supported at this time so the Type field is not exposed. + +_Appears in:_ +- [GatewayDeploymentSpec](#gatewaydeploymentspec) + +| Field | Description | +| --- | --- | +| `rollingUpdate` _[RollingUpdateDeployment](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#rollingupdatedeployment-v1-apps)_ | (Optional) | + + +### GatewayKind + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [Deployment DaemonSet] + + +_Appears in:_ +- [GatewayClassSpec](#gatewayclassspec) + +| Value | Description | +| --- | --- | +| `Deployment` | | +| `DaemonSet` | | + + +### GatewayService + + + +GatewayService allows customization of the Services that front Gateways. + +_Appears in:_ +- [GatewayClassSpec](#gatewayclassspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[GatewayServiceSpec](#gatewayservicespec)_ | (Optional) | + + +### GatewayServiceSpec + + + +GatewayServiceSpec allows customization of the services that front gateway deployments. + +The LoadBalancer fields allow customization of the corresponding fields in the Kubernetes +ServiceSpec. These can be used for some cloud-independent control of the external load balancer +that is provisioned for each Gateway. For finer-grained cloud-specific control please use +the Metadata.Annotations field in GatewayService. + +_Appears in:_ +- [GatewayService](#gatewayservice) + +| Field | Description | +| --- | --- | +| `loadBalancerClass` _string_ | (Optional) | +| `allocateLoadBalancerNodePorts` _boolean_ | (Optional) | +| `loadBalancerSourceRanges` _string array_ | (Optional) | +| `loadBalancerIP` _string_ | (Optional) | +| `patch` _[KubernetesPatchSpec](#kubernetespatchspec)_ | (Optional) Patch allows the Service for a gateway to be patched in ways that aren't more explicitly supported by the fields above. For example, the following YAML could be used to set the Service's healthCheckNodePort: patch: type: StrategicMerge value: spec: healthCheckNodePort: 12345 | + + +### Goldmane + + + + + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `Goldmane` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[GoldmaneSpec](#goldmanespec)_ | | +| `status` _[GoldmaneStatus](#goldmanestatus)_ | | + + +### GoldmaneDeployment + + + +GoldmaneDeployment is the configuration for the goldmane Deployment. + +_Appears in:_ +- [GoldmaneSpec](#goldmanespec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[GoldmaneDeploymentSpec](#goldmanedeploymentspec)_ | (Optional) Spec is the specification of the goldmane Deployment. | + + +### GoldmaneDeploymentContainer + + + + + +_Appears in:_ +- [GoldmaneDeploymentPodSpec](#goldmanedeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### GoldmaneDeploymentPodSpec + + + +GoldmaneDeploymentPodSpec is the goldmane Deployment's PodSpec. + +_Appears in:_ +- [GoldmaneDeploymentPodTemplateSpec](#goldmanedeploymentpodtemplatespec) + +| Field | Description | +| --- | --- | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) Affinity is a group of affinity scheduling rules for the goldmane pods. | +| `containers` _[GoldmaneDeploymentContainer](#goldmanedeploymentcontainer) array_ | (Optional) Containers is a list of goldmane containers. If specified, this overrides the specified EGW Deployment containers. If omitted, the goldmane Deployment will use its default values for its containers. | +| `nodeSelector` _object (keys:string, values:string)_ | (Optional) NodeSelector gives more control over the nodes where the goldmane pods will run on. | +| `terminationGracePeriodSeconds` _integer_ | (Optional) TerminationGracePeriodSeconds defines the termination grace period of the goldmane pods in seconds. | +| `topologySpreadConstraints` _[TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#topologyspreadconstraint-v1-core) array_ | (Optional) TopologySpreadConstraints describes how a group of pods ought to spread across topology domains. Scheduler will schedule pods in a way which abides by the constraints. All topologySpreadConstraints are ANDed. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) Tolerations is the goldmane pod's tolerations. If specified, this overrides any tolerations that may be set on the goldmane Deployment. If omitted, the goldmane Deployment will use its default value for tolerations. | +| `priorityClassName` _string_ | (Optional) PriorityClassName allows to specify a PriorityClass resource to be used. | + + +### GoldmaneDeploymentPodTemplateSpec + + + +GoldmaneDeploymentPodTemplateSpec is the goldmane Deployment's PodTemplateSpec + +_Appears in:_ +- [GoldmaneDeploymentSpec](#goldmanedeploymentspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[GoldmaneDeploymentPodSpec](#goldmanedeploymentpodspec)_ | (Optional) Spec is the goldmane Deployment's PodSpec. | + + +### GoldmaneDeploymentSpec + + + +GoldmaneDeploymentSpec defines configuration for the goldmane Deployment. + +_Appears in:_ +- [GoldmaneDeployment](#goldmanedeployment) + +| Field | Description | +| --- | --- | +| `minReadySeconds` _integer_ | (Optional) MinReadySeconds is the minimum number of seconds for which a newly created Deployment pod should be ready without any of its container crashing, for it to be considered available. If specified, this overrides any minReadySeconds value that may be set on the goldmane Deployment. If omitted, the goldmane Deployment will use its default value for minReadySeconds. | +| `template` _[GoldmaneDeploymentPodTemplateSpec](#goldmanedeploymentpodtemplatespec)_ | (Optional) Template describes the goldmane Deployment pod that will be created. | +| `strategy` _[GoldmaneDeploymentStrategy](#goldmanedeploymentstrategy)_ | (Optional) The deployment strategy to use to replace existing pods with new ones. | + + +### GoldmaneDeploymentStrategy + + + + + +_Appears in:_ +- [GoldmaneDeploymentSpec](#goldmanedeploymentspec) + +| Field | Description | +| --- | --- | +| `rollingUpdate` _[RollingUpdateDeployment](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#rollingupdatedeployment-v1-apps)_ | (Optional) Rolling update config params. Present only if DeploymentStrategyType = RollingUpdate. to be. | + + +### GoldmaneSpec + + + + + +_Appears in:_ +- [Goldmane](#goldmane) + +| Field | Description | +| --- | --- | +| `goldmaneDeployment` _[GoldmaneDeployment](#goldmanedeployment)_ | | +| `metricsPort` _integer_ | (Optional) MetricsPort configures the port that Goldmane uses to serve Prometheus metrics. When set to a non-zero value, Goldmane will expose a /metrics endpoint on the given port. Set to zero to disable metrics. If omitted, metrics are disabled. | + + +### GoldmaneStatus + + + +GoldmaneStatus defines the observed state of Goldmane + +_Appears in:_ +- [Goldmane](#goldmane) + +| Field | Description | +| --- | --- | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#condition-v1-meta) array_ | (Optional) Conditions represents the latest observed set of conditions for the component. A component may be one or more of Ready, Progressing, Degraded or other customer types. | + + +### GuardianDeployment + + + +GuardianDeployment is the configuration for the guardian Deployment. + +_Appears in:_ +- [ManagementClusterConnectionSpec](#managementclusterconnectionspec) + +| Field | Description | +| --- | --- | +| `spec` _[GuardianDeploymentSpec](#guardiandeploymentspec)_ | (Optional) Spec is the specification of the guardian Deployment. | + + +### GuardianDeploymentContainer + + + +GuardianDeploymentContainer is a guardian Deployment container. + +_Appears in:_ +- [GuardianDeploymentPodSpec](#guardiandeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the guardian Deployment container by name.
Supported values are: tigera-guardian | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named guardian Deployment container's resources. If omitted, the guardian Deployment will use its default value for this container's resources. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### GuardianDeploymentInitContainer + + + +GuardianDeploymentInitContainer is a guardian Deployment init container. + +_Appears in:_ +- [GuardianDeploymentPodSpec](#guardiandeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the guardian Deployment init container by name. | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named guardian Deployment init container's resources. If omitted, the guardian Deployment will use its default value for this init container's resources. | + + +### GuardianDeploymentPodSpec + + + +GuardianDeploymentPodSpec is the guardian Deployment's PodSpec. + +_Appears in:_ +- [GuardianDeploymentPodTemplateSpec](#guardiandeploymentpodtemplatespec) + +| Field | Description | +| --- | --- | +| `initContainers` _[GuardianDeploymentInitContainer](#guardiandeploymentinitcontainer) array_ | (Optional) InitContainers is a list of guardian init containers. If specified, this overrides the specified guardian Deployment init containers. If omitted, the guardian Deployment will use its default values for its init containers. | +| `containers` _[GuardianDeploymentContainer](#guardiandeploymentcontainer) array_ | (Optional) Containers is a list of guardian containers. If specified, this overrides the specified guardian Deployment containers. If omitted, the guardian Deployment will use its default values for its containers. | + + +### GuardianDeploymentPodTemplateSpec + + + +GuardianDeploymentPodTemplateSpec is the guardian Deployment's PodTemplateSpec + +_Appears in:_ +- [GuardianDeploymentSpec](#guardiandeploymentspec) + +| Field | Description | +| --- | --- | +| `spec` _[GuardianDeploymentPodSpec](#guardiandeploymentpodspec)_ | (Optional) Spec is the guardian Deployment's PodSpec. | + + +### GuardianDeploymentSpec + + + +GuardianDeploymentSpec defines configuration for the guardian Deployment. + +_Appears in:_ +- [GuardianDeployment](#guardiandeployment) + +| Field | Description | +| --- | --- | +| `template` _[GuardianDeploymentPodTemplateSpec](#guardiandeploymentpodtemplatespec)_ | (Optional) Template describes the guardian Deployment pod that will be created. | + + +### HTTPProbe + + + +HTTPProbe defines the HTTP probe configuration for Egress Gateway. + +_Appears in:_ +- [EgressGatewayFailureDetection](#egressgatewayfailuredetection) + +| Field | Description | +| --- | --- | +| `urls` _string array_ | URLs define the list of HTTP probe URLs. Egress Gateway will probe each URL periodically.If all probes fail, Egress Gateway will report non-ready. | +| `intervalSeconds` _integer_ | (Optional) IntervalSeconds defines the interval of HTTP probes. Used when URLs is non-empty.
Default: 10 | +| `timeoutSeconds` _integer_ | (Optional) TimeoutSeconds defines the timeout value of HTTP probes. Used when URLs is non-empty.
Default: 30 | + + +### HostPortsType + +_Underlying type:_ _string_ + +HostPortsType specifies host port support. + +One of: Enabled, Disabled + +_Appears in:_ +- [CalicoNetworkSpec](#caliconetworkspec) + +| Value | Description | +| --- | --- | +| `Enabled` | | +| `Disabled` | | + + +### HostScope + +_Underlying type:_ _string_ + +HostScope determines the set of hosts that forward logs to a given store. + +_Validation:_ +- Enum: [All NonClusterOnly] + + +_Appears in:_ +- [S3StoreSpec](#s3storespec) +- [SplunkStoreSpec](#splunkstorespec) +- [SyslogStoreSpec](#syslogstorespec) + +| Value | Description | +| --- | --- | +| `All` | | +| `NonClusterOnly` | | + + +### ICMPProbe + + + +ICMPProbe defines the ICMP probe configuration for Egress Gateway. + +_Appears in:_ +- [EgressGatewayFailureDetection](#egressgatewayfailuredetection) + +| Field | Description | +| --- | --- | +| `ips` _string array_ | IPs define the list of ICMP probe IPs. Egress Gateway will probe each IP periodically. If all probes fail, Egress Gateway will report non-ready. | +| `intervalSeconds` _integer_ | (Optional) IntervalSeconds defines the interval of ICMP probes. Used when IPs is non-empty.
Default: 5 | +| `timeoutSeconds` _integer_ | (Optional) TimeoutSeconds defines the timeout value of ICMP probes. Used when IPs is non-empty.
Default: 15 | + + +### IPAMPluginType + +_Underlying type:_ _string_ + + + +_Appears in:_ +- [IPAMSpec](#ipamspec) + +| Value | Description | +| --- | --- | +| `Calico` | | +| `HostLocal` | | +| `AmazonVPC` | | +| `AzureVNET` | | + + +### IPAMSpec + + + +IPAMSpec contains configuration for pod IP address management. + +_Appears in:_ +- [CNISpec](#cnispec) + +| Field | Description | +| --- | --- | +| `type` _[IPAMPluginType](#ipamplugintype)_ | Specifies the IPAM plugin that will be used in the Calico or Calico Enterprise installation. * For CNI Plugin Calico, this field defaults to Calico. * For CNI Plugin GKE, this field defaults to HostLocal. * For CNI Plugin AzureVNET, this field defaults to AzureVNET. * For CNI Plugin AmazonVPC, this field defaults to AmazonVPC. The IPAM plugin is installed and configured only if the CNI plugin is set to Calico, for all other values of the CNI plugin the plugin binaries and CNI config is a dependency that is expected to be installed separately.
Default: Calico | + + +### IPPool + + + + + +_Appears in:_ +- [CalicoNetworkSpec](#caliconetworkspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is the name of the IP pool. If omitted, this will be generated. | +| `cidr` _string_ | CIDR contains the address range for the IP Pool in classless inter-domain routing format. | +| `encapsulation` _[EncapsulationType](#encapsulationtype)_ | (Optional) Encapsulation specifies the encapsulation type that will be used with the IP Pool.
Default: IPIP | +| `natOutgoing` _[NATOutgoingType](#natoutgoingtype)_ | (Optional) NATOutgoing specifies if NAT will be enabled or disabled for outgoing traffic.
Default: Enabled | +| `nodeSelector` _string_ | (Optional) NodeSelector specifies the node selector that will be set for the IP Pool.
Default: 'all()' | +| `blockSize` _integer_ | (Optional) BlockSize specifies the CIDR prefex length to use when allocating per-node IP blocks from the main IP pool CIDR.
Default: 26 (IPv4), 122 (IPv6) | +| `disableBGPExport` _boolean_ | (Optional) DisableBGPExport specifies whether routes from this IP pool's CIDR are exported over BGP.
Default: false | +| `disableNewAllocations` _boolean_ | DisableNewAllocations specifies whether or not new IP allocations are allowed from this pool. This is useful when you want to prevent new pods from receiving IP addresses from this pool, without impacting any existing pods that have already been assigned addresses from this pool. | +| `allowedUses` _[IPPoolAllowedUse](#ippoolalloweduse) array_ | AllowedUse controls what the IP pool will be used for. If not specified or empty, defaults to ["Tunnel", "Workload"] for back-compatibility | +| `assignmentMode` _[AssignmentMode](#assignmentmode)_ | AssignmentMode determines if IP addresses from this pool should be assigned automatically or on request only | + + +### IPPoolAllowedUse + +_Underlying type:_ _string_ + + + +_Appears in:_ +- [IPPool](#ippool) + +| Value | Description | +| --- | --- | +| `Workload` | | +| `Tunnel` | | +| `LoadBalancer` | | + + +### Image + + + + + +_Appears in:_ +- [ImageSetSpec](#imagesetspec) + +| Field | Description | +| --- | --- | +| `image` _string_ | Image is an image that the operator deploys and instead of using the built in tag the operator will use the Digest for the image identifier. The value should be the *original* image name without registry or tag or digest. For the image `docker.io/calico/node:v3.17.1` it should be represented as `calico/node` The "Installation" spec allows defining custom image registries, paths or prefixes. Even for custom images such as example.com/custompath/customprefix-calico-node:v3.17.1, this value should still be `calico/node`. | +| `digest` _string_ | Digest is the image identifier that will be used for the Image. The field should not include a leading `@` and must be prefixed with `sha256:`. | + + +### ImageSet + + + +ImageSet is used to specify image digests for the images that the operator deploys. +The name of the ImageSet is expected to be in the format `-`. +The `variant` used is `enterprise` if the InstallationSpec Variant is +`CalicoEnterprise` or `TigeraSecureEnterprise`, otherwise it is `calico`. +The `release` must match the version of the variant that the operator is built to deploy, +this version can be obtained by passing the `--version` flag to the operator binary. + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `ImageSet` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[ImageSetSpec](#imagesetspec)_ | | + + +### ImageSetSpec + + + +ImageSetSpec defines the desired state of ImageSet. + +_Appears in:_ +- [ImageSet](#imageset) + +| Field | Description | +| --- | --- | +| `images` _[Image](#image) array_ | Images is the list of images to use digests. All images that the operator will deploy must be specified. | + + +### Impersonation + + + +Impersonation defines the rules for allowing impersonation. + +_Appears in:_ +- [ManagementClusterConnectionSpec](#managementclusterconnectionspec) + +| Field | Description | +| --- | --- | +| `users` _string array_ | (Optional) Users is a list of users that can be impersonated. An empty list infers all users can be impersonated, a null value means none. | +| `groups` _string array_ | (Optional) Groups is a list of group names that can be impersonated. An empty list infers all groups can be impersonated, a null values means none. | +| `serviceAccounts` _string array_ | (Optional) ServiceAccounts is a list of service account names that can be impersonated. An empty list infers all service accounts can be impersonated, a null values means none. | + + + + +### Installation + + + +Installation configures an installation of Calico or Calico Enterprise. At most one instance +of this resource is supported. It must be named "default". The Installation API installs core networking +and network policy components, and provides general install-time configuration. + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `Installation` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[InstallationSpec](#installationspec)_ | Specification of the desired state for the Calico or Calico Enterprise installation. | +| `status` _[InstallationStatus](#installationstatus)_ | Most recently observed state for the Calico or Calico Enterprise installation. | + + +### InstallationSpec + + + +InstallationSpec defines configuration for a Calico or Calico Enterprise installation. + +_Appears in:_ +- [Installation](#installation) +- [InstallationStatus](#installationstatus) + +| Field | Description | +| --- | --- | +| `variant` _[ProductVariant](#productvariant)_ | (Optional) Variant is the product to install - one of Calico or CalicoEnterprise. TigeraSecureEnterprise is also accepted as a deprecated alias for CalicoEnterprise.
Default: Calico | +| `registry` _string_ | (Optional) Registry is the default Docker registry used for component Docker images. If specified then the given value must end with a slash character (`/`) and all images will be pulled from this registry. If not specified then the default registries will be used. A special case value, UseDefault, is supported to explicitly specify the default registries will be used. Image format: `/:` This option allows configuring the `` portion of the above format. | +| `imagePath` _string_ | (Optional) ImagePath allows for the path part of an image to be specified. If specified then the specified value will be used as the image path for each image. If not specified or empty, the default for each image will be used. A special case value, UseDefault, is supported to explicitly specify the default image path will be used for each image. Image format: `/:` This option allows configuring the `` portion of the above format. | +| `imagePrefix` _string_ | (Optional) ImagePrefix allows for the prefix part of an image to be specified. If specified then the given value will be used as a prefix on each image. If not specified or empty, no prefix will be used. A special case value, UseDefault, is supported to explicitly specify the default image prefix will be used for each image. Image format: `/:` This option allows configuring the `` portion of the above format. | +| `imagePullSecrets` _[LocalObjectReference](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#localobjectreference-v1-core) array_ | (Optional) ImagePullSecrets is an array of references to container registry pull secrets to use. These are applied to all images to be pulled. | +| `imagePullPolicy` _[PullPolicy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#pullpolicy-v1-core)_ | (Optional) ImagePullPolicy is the pull policy applied to containers in pods rendered by the operator that do not explicitly set their own pull policy. If unset, defaults to IfNotPresent. This is useful in air-gapped environments where images are pre-loaded onto nodes and must not be re-pulled from a remote registry. | +| `kubernetesProvider` _[Provider](#provider)_ | (Optional) KubernetesProvider specifies a particular provider of the Kubernetes platform and enables provider-specific configuration. If the specified value is empty, the Operator will attempt to automatically determine the current provider. If the specified value is not empty, the Operator will still attempt auto-detection, but will additionally compare the auto-detected value to the specified value to confirm they match. | +| `cni` _[CNISpec](#cnispec)_ | (Optional) CNI specifies the CNI that will be used by this installation. | +| `calicoNetwork` _[CalicoNetworkSpec](#caliconetworkspec)_ | (Optional) CalicoNetwork specifies networking configuration options for Calico. | +| `typhaAffinity` _[TyphaAffinity](#typhaaffinity)_ | (Optional) Deprecated. Please use Installation.Spec.TyphaDeployment instead. TyphaAffinity allows configuration of node affinity characteristics for Typha pods. | +| `controlPlaneNodeSelector` _object (keys:string, values:string)_ | (Optional) ControlPlaneNodeSelector is used to select control plane nodes on which to run Calico components. This is globally applied to all resources created by the operator excluding daemonsets. | +| `controlPlaneTolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) ControlPlaneTolerations specify tolerations which are then globally applied to all resources created by the operator. | +| `controlPlaneReplicas` _integer_ | (Optional) ControlPlaneReplicas defines how many replicas of the control plane core components will be deployed. This field applies to all control plane components that support High Availability. Defaults to 2. | +| `nodeMetricsPort` _integer_ | (Optional) NodeMetricsPort specifies which port calico/node serves prometheus metrics on. By default, metrics are not enabled. If specified, this overrides any FelixConfiguration resources which may exist. If omitted, then prometheus metrics may still be configured through FelixConfiguration. | +| `typhaMetricsPort` _integer_ | (Optional) TyphaMetricsPort specifies which port calico/typha serves prometheus metrics on. By default, metrics are not enabled. | +| `flexVolumePath` _string_ | (Optional) FlexVolumePath optionally specifies a custom path for FlexVolume. If not specified, FlexVolume will be enabled by default. If set to 'None', FlexVolume will be disabled. The default is based on the kubernetesProvider. | +| `kubeletVolumePluginPath` _string_ | (Optional) KubeletVolumePluginPath optionally specifies enablement of Calico CSI plugin. If not specified, CSI will be enabled by default. If set to 'None', CSI will be disabled.
Default: /var/lib/kubelet | +| `nodeUpdateStrategy` _[DaemonSetUpdateStrategy](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#daemonsetupdatestrategy-v1-apps)_ | (Optional) NodeUpdateStrategy can be used to customize the desired update strategy, such as the MaxUnavailable field. | +| `componentResources` _[ComponentResource](#componentresource) array_ | (Optional) Deprecated. Please use CalicoNodeDaemonSet, TyphaDeployment, and KubeControllersDeployment. ComponentResources can be used to customize the resource requirements for each component. Node, Typha, and KubeControllers are supported for installations. | +| `certificateManagement` _[CertificateManagement](#certificatemanagement)_ | (Optional) CertificateManagement configures pods to submit a CertificateSigningRequest to the certificates.k8s.io/v1 API in order to obtain TLS certificates. This feature requires that you bring your own CSR signing and approval process, otherwise pods will be stuck during initialization. | +| `tlsCipherSuites` _[TLSCipherSuites](#tlsciphersuites)_ | (Optional) TLSCipherSuites defines the cipher suite list that the TLS protocol should use during secure communication. | +| `nonPrivileged` _[NonPrivilegedType](#nonprivilegedtype)_ | (Optional) Deprecated. NonPrivileged is deprecated and will be removed from the API in a future release. Enabling this field is not supported and will cause errors. NonPrivileged configures Calico to be run in non-privileged containers as non-root users where possible. | +| `calicoNodeDaemonSet` _[CalicoNodeDaemonSet](#caliconodedaemonset)_ | (Optional) CalicoNodeDaemonSet configures the calico-node DaemonSet. If used in conjunction with the deprecated ComponentResources, then these overrides take precedence. | +| `csiNodeDriverDaemonSet` _[CSINodeDriverDaemonSet](#csinodedriverdaemonset)_ | (Optional) CSINodeDriverDaemonSet configures the csi-node-driver DaemonSet. | +| `calicoKubeControllersDeployment` _[CalicoKubeControllersDeployment](#calicokubecontrollersdeployment)_ | (Optional) CalicoKubeControllersDeployment configures the calico-kube-controllers Deployment. If used in conjunction with the deprecated ComponentResources, then these overrides take precedence. | +| `typhaDeployment` _[TyphaDeployment](#typhadeployment)_ | (Optional) TyphaDeployment configures the typha Deployment. If used in conjunction with the deprecated ComponentResources or TyphaAffinity, then these overrides take precedence. | +| `typhaPodDisruptionBudget` _[PodDisruptionBudgetOverride](#poddisruptionbudgetoverride)_ | (Optional) TyphaPodDisruptionBudget configures the PodDisruptionBudget for the calico-typha Deployment. Fields left unset fall back to the operator's defaults. The PDB's selector is managed by the operator and cannot be overridden. | +| `calicoWindowsUpgradeDaemonSet` _[CalicoWindowsUpgradeDaemonSet](#calicowindowsupgradedaemonset)_ | Deprecated. The CalicoWindowsUpgradeDaemonSet is deprecated and will be removed from the API in the future. CalicoWindowsUpgradeDaemonSet configures the calico-windows-upgrade DaemonSet. | +| `calicoNodeWindowsDaemonSet` _[CalicoNodeWindowsDaemonSet](#caliconodewindowsdaemonset)_ | CalicoNodeWindowsDaemonSet configures the calico-node-windows DaemonSet. | +| `fipsMode` _[FIPSMode](#fipsmode)_ | (Optional) Deprecated. FIPS mode is no longer supported. Setting fipsMode to Enabled marks the installation degraded. | +| `logging` _[Logging](#logging)_ | (Optional) Logging Configuration for Components | +| `windowsNodes` _[WindowsNodeSpec](#windowsnodespec)_ | (Optional) Windows Configuration | +| `serviceCIDRs` _string array_ | (Optional) Kubernetes Service CIDRs. Specifying this is required when using Calico for Windows. | +| `azure` _[Azure](#azure)_ | (Optional) Azure is used to configure azure provider specific options. | +| `proxy` _[Proxy](#proxy)_ | (Optional) Proxy is used to configure the HTTP(S) proxy settings that will be applied to Tigera containers that connect to destinations outside the cluster. It is expected that NO_PROXY is configured such that destinations within the cluster (including the API server) are exempt from proxying. | +| `networkPolicy` _[NetworkPolicySpec](#networkpolicyspec)_ | (Optional) NetworkPolicy configures how the operator manages the NetworkPolicies and GlobalNetworkPolicies it installs to protect the Calico components it manages. | + + +### InstallationStatus + + + +InstallationStatus defines the observed state of the Calico or Calico Enterprise installation. + +_Appears in:_ +- [Installation](#installation) + +| Field | Description | +| --- | --- | +| `variant` _[ProductVariant](#productvariant)_ | Variant is the most recently observed installed variant - one of Calico or CalicoEnterprise. TigeraSecureEnterprise is a deprecated alias for CalicoEnterprise. | +| `mtu` _integer_ | MTU is the most recently observed value for pod network MTU. This may be an explicitly configured value, or based on Calico's native auto-detetion. | +| `imageSet` _string_ | (Optional) ImageSet is the name of the ImageSet being used, if there is an ImageSet that is being used. If an ImageSet is not being used then this will not be set. | +| `computed` _[InstallationSpec](#installationspec)_ | (Optional) Computed is the final installation including overlaid resources. | +| `calicoVersion` _string_ | CalicoVersion shows the current running version of calico. CalicoVersion along with Variant is needed to know the exact version deployed. | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#condition-v1-meta) array_ | (Optional) Conditions represents the latest observed set of conditions for the component. A component may be one or more of Ready, Progressing, Degraded or other customer types. | + + +### IntrusionDetection + + + +IntrusionDetection installs the components required for Tigera intrusion detection. At most one instance +of this resource is supported. It must be named "tigera-secure". + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `IntrusionDetection` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[IntrusionDetectionSpec](#intrusiondetectionspec)_ | Specification of the desired state for Tigera intrusion detection. | +| `status` _[IntrusionDetectionStatus](#intrusiondetectionstatus)_ | Most recently observed state for Tigera intrusion detection. | + + +### IntrusionDetectionComponentName + +_Underlying type:_ _string_ + + + +_Appears in:_ +- [IntrusionDetectionComponentResource](#intrusiondetectioncomponentresource) + +| Value | Description | +| --- | --- | +| `DeepPacketInspection` | | + + +### IntrusionDetectionComponentResource + + + +The ComponentResource struct associates a ResourceRequirements with a component by name + +_Appears in:_ +- [IntrusionDetectionSpec](#intrusiondetectionspec) + +| Field | Description | +| --- | --- | +| `componentName` _[IntrusionDetectionComponentName](#intrusiondetectioncomponentname)_ | ComponentName is an enum which identifies the component | +| `resourceRequirements` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | ResourceRequirements allows customization of limits and requests for compute resources such as cpu and memory. | + + +### IntrusionDetectionControllerDeployment + + + +IntrusionDetectionControllerDeployment is the configuration for the IntrusionDetectionController Deployment. + +_Appears in:_ +- [IntrusionDetectionSpec](#intrusiondetectionspec) + +| Field | Description | +| --- | --- | +| `spec` _[IntrusionDetectionControllerDeploymentSpec](#intrusiondetectioncontrollerdeploymentspec)_ | (Optional) Spec is the specification of the IntrusionDetectionController Deployment. | + + +### IntrusionDetectionControllerDeploymentContainer + + + +IntrusionDetectionControllerDeploymentContainer is a IntrusionDetectionController Deployment container. + +_Appears in:_ +- [IntrusionDetectionControllerDeploymentPodSpec](#intrusiondetectioncontrollerdeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the IntrusionDetectionController Deployment container by name.
Supported values are: controller, webhooks-processor | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named IntrusionDetectionController Deployment container's resources. If omitted, the IntrusionDetection Deployment will use its default value for this container's resources. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### IntrusionDetectionControllerDeploymentInitContainer + + + +IntrusionDetectionControllerDeploymentInitContainer is a IntrusionDetectionController Deployment init container. + +_Appears in:_ +- [IntrusionDetectionControllerDeploymentPodSpec](#intrusiondetectioncontrollerdeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the IntrusionDetectionController Deployment init container by name.
Supported values are: intrusion-detection-tls-key-cert-provisioner | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named IntrusionDetectionController Deployment init container's resources. If omitted, the IntrusionDetectionController Deployment will use its default value for this init container's resources. | + + +### IntrusionDetectionControllerDeploymentPodSpec + + + +IntrusionDetectionControllerDeploymentPodSpec is the IntrusionDetectionController Deployment's PodSpec. + +_Appears in:_ +- [IntrusionDetectionControllerDeploymentPodTemplateSpec](#intrusiondetectioncontrollerdeploymentpodtemplatespec) + +| Field | Description | +| --- | --- | +| `initContainers` _[IntrusionDetectionControllerDeploymentInitContainer](#intrusiondetectioncontrollerdeploymentinitcontainer) array_ | (Optional) InitContainers is a list of IntrusionDetectionController init containers. If specified, this overrides the specified IntrusionDetectionController Deployment init containers. If omitted, the IntrusionDetectionController Deployment will use its default values for its init containers. | +| `containers` _[IntrusionDetectionControllerDeploymentContainer](#intrusiondetectioncontrollerdeploymentcontainer) array_ | (Optional) Containers is a list of IntrusionDetectionController containers. If specified, this overrides the specified IntrusionDetectionController Deployment containers. If omitted, the IntrusionDetectionController Deployment will use its default values for its containers. | + + +### IntrusionDetectionControllerDeploymentPodTemplateSpec + + + +IntrusionDetectionControllerDeploymentPodTemplateSpec is the IntrusionDetectionController Deployment's PodTemplateSpec + +_Appears in:_ +- [IntrusionDetectionControllerDeploymentSpec](#intrusiondetectioncontrollerdeploymentspec) + +| Field | Description | +| --- | --- | +| `spec` _[IntrusionDetectionControllerDeploymentPodSpec](#intrusiondetectioncontrollerdeploymentpodspec)_ | (Optional) Spec is the IntrusionDetectionController Deployment's PodSpec. | + + +### IntrusionDetectionControllerDeploymentSpec + + + +IntrusionDetectionControllerDeploymentSpec defines configuration for the IntrusionDetectionController Deployment. + +_Appears in:_ +- [IntrusionDetectionControllerDeployment](#intrusiondetectioncontrollerdeployment) + +| Field | Description | +| --- | --- | +| `template` _[IntrusionDetectionControllerDeploymentPodTemplateSpec](#intrusiondetectioncontrollerdeploymentpodtemplatespec)_ | (Optional) Template describes the IntrusionDetectionController Deployment pod that will be created. | + + +### IntrusionDetectionSpec + + + +IntrusionDetectionSpec defines the desired state of Tigera intrusion detection capabilities. + +_Appears in:_ +- [IntrusionDetection](#intrusiondetection) + +| Field | Description | +| --- | --- | +| `componentResources` _[IntrusionDetectionComponentResource](#intrusiondetectioncomponentresource) array_ | (Optional) ComponentResources can be used to customize the resource requirements for each component. Only DeepPacketInspection is supported for this spec. | +| `anomalyDetection` _[AnomalyDetectionSpec](#anomalydetectionspec)_ | (Optional) AnomalyDetection is now deprecated, and configuring it has no effect. | +| `intrusionDetectionControllerDeployment` _[IntrusionDetectionControllerDeployment](#intrusiondetectioncontrollerdeployment)_ | (Optional) IntrusionDetectionControllerDeployment configures the IntrusionDetection Controller Deployment. | +| `deepPacketInspectionDaemonset` _[DeepPacketInspectionDaemonset](#deeppacketinspectiondaemonset)_ | (Optional) DeepPacketInspectionDaemonset configures the DPI Daemonset | + + +### IntrusionDetectionStatus + + + +IntrusionDetectionStatus defines the observed state of Tigera intrusion detection capabilities. + +_Appears in:_ +- [IntrusionDetection](#intrusiondetection) + +| Field | Description | +| --- | --- | +| `state` _string_ | State provides user-readable status. | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#condition-v1-meta) array_ | (Optional) Conditions represents the latest observed set of conditions for the component. A component may be one or more of Ready, Progressing, Degraded or other customer types. | + + +### Istio + + + +Istio is the Schema for the istios API + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `Istio` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[IstioSpec](#istiospec)_ | | +| `status` _[IstioStatus](#istiostatus)_ | | + + +### IstioCNIDaemonset + + + +IstioCNIDaemonset defines customized settings for the Istio CNI plugin. + +_Appears in:_ +- [IstioSpec](#istiospec) + +| Field | Description | +| --- | --- | +| `spec` _[IstioCNIDaemonsetSpec](#istiocnidaemonsetspec)_ | (Optional) Spec allows users to specify custom fields for the Istio CNI Daemonset. | + + +### IstioCNIDaemonsetPodSpec + + + +IstioCNIDaemonsetPodSpec defines the pod spec for customizing the Istio CNI Daemonset. + +_Appears in:_ +- [IstioCNIDaemonsetSpecTemplate](#istiocnidaemonsetspectemplate) + +| Field | Description | +| --- | --- | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) Affinity specifies the affinity for the deployment. | +| `nodeSelector` _object (keys:string, values:string)_ | (Optional) NodeSelector specifies the node affinity for the deployment. | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources specifies the compute resources required for the deployment. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) Tolerations specifies the tolerations for the deployment. | + + +### IstioCNIDaemonsetSpec + + + +IstioCNIDaemonsetSpec defines the spec for customizing the Istio CNI Daemonset. + +_Appears in:_ +- [IstioCNIDaemonset](#istiocnidaemonset) + +| Field | Description | +| --- | --- | +| `template` _[IstioCNIDaemonsetSpecTemplate](#istiocnidaemonsetspectemplate)_ | (Optional) Template allows users to specify custom fields for the Istio CNI Daemonset. | + + +### IstioCNIDaemonsetSpecTemplate + + + +IstioCNIDaemonsetSpecTemplate defines the template for customizing the Istio CNI Daemonset. + +_Appears in:_ +- [IstioCNIDaemonsetSpec](#istiocnidaemonsetspec) + +| Field | Description | +| --- | --- | +| `spec` _[IstioCNIDaemonsetPodSpec](#istiocnidaemonsetpodspec)_ | (Optional) Spec allows users to specify custom fields for the Istio CNI Daemonset. | + + +### IstioSpec + + + +IstioSpec defines the desired state of Istio + +_Appears in:_ +- [Istio](#istio) + +| Field | Description | +| --- | --- | +| `istiod` _[IstiodDeployment](#istioddeployment)_ | (Optional) IstiodDeployment defines the resource requirements and node selector for the Istio deployment. | +| `istioCNI` _[IstioCNIDaemonset](#istiocnidaemonset)_ | (Optional) IstioCNIDaemonset defines the resource requirements for the Istio CNI plugin. | +| `ztunnel` _[ZTunnelDaemonset](#ztunneldaemonset)_ | (Optional) ZTunnelDaemonset defines the resource requirements for the ZTunnelDaemonset component. | +| `waypointLogging` _[LogCollectionStatusType](#logcollectionstatustype)_ | (Optional) WaypointLogging controls whether L7 logging is enabled on every Gateway using the istio-waypoint GatewayClass. When Enabled (the default), the operator injects an l7-collector sidecar into each waypoint pod via class-level defaults. When Disabled, no sidecar is injected and the associated EnvoyFilters are not created. Allowed values are Enabled or Disabled. | +| `dscpMark` _[DSCP](#dscp)_ | (Optional) DSCPMark define the value of the DSCP mark done by Felix and recognised by Istio CNI for Transparent NetworkPolicies. | + + +### IstioStatus + + + +IstioStatus defines the observed state of Istio + +_Appears in:_ +- [Istio](#istio) + +| Field | Description | +| --- | --- | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#condition-v1-meta) array_ | Conditions represents the latest observed set of conditions for the component. A component may be one or more of Ready, Progressing, Degraded or other customer types. | + + +### IstiodDeployment + + + +IstiodDeployment defines customized settings for the Istio deployment. + +_Appears in:_ +- [IstioSpec](#istiospec) + +| Field | Description | +| --- | --- | +| `spec` _[IstiodDeploymentSpec](#istioddeploymentspec)_ | (Optional) Spec allows users to specify custom fields for the Istiod Deployment. | + + +### IstiodDeploymentPodSpec + + + +IstiodDeploymentPodSpec defines the pod spec for customizing the Istiod Deployment. + +_Appears in:_ +- [IstiodDeploymentSpecTemplate](#istioddeploymentspectemplate) + +| Field | Description | +| --- | --- | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) Affinity specifies the affinity for the deployment. | +| `nodeSelector` _object (keys:string, values:string)_ | (Optional) NodeSelector specifies the node affinity for the deployment. | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources specifies the compute resources required for the deployment. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) Tolerations specifies the tolerations for the deployment. | + + +### IstiodDeploymentSpec + + + +IstiodDeploymentSpec defines the spec for customizing the Istiod Deployment. + +_Appears in:_ +- [IstiodDeployment](#istioddeployment) + +| Field | Description | +| --- | --- | +| `template` _[IstiodDeploymentSpecTemplate](#istioddeploymentspectemplate)_ | (Optional) Template allows users to specify custom fields for the Istiod Deployment. | + + +### IstiodDeploymentSpecTemplate + + + +IstiodDeploymentSpecTemplate defines the template for customizing the Istiod Deployment. + +_Appears in:_ +- [IstiodDeploymentSpec](#istioddeploymentspec) + +| Field | Description | +| --- | --- | +| `spec` _[IstiodDeploymentPodSpec](#istioddeploymentpodspec)_ | (Optional) Spec allows users to specify custom fields for the Istiod Deployment. | + + +### KubeProxyManagementType + +_Underlying type:_ _string_ + +KubeProxyManagementType specifies whether kube-proxy management is enabled. + +_Appears in:_ +- [CalicoNetworkSpec](#caliconetworkspec) + +| Value | Description | +| --- | --- | +| `Enabled` | | +| `Disabled` | | + + +### KubernetesAutodetectionMethod + +_Underlying type:_ _string_ + +KubernetesAutodetectionMethod is a method of detecting an IP address based on the Kubernetes API. + +One of: NodeInternalIP + +_Appears in:_ +- [NodeAddressAutodetection](#nodeaddressautodetection) + +| Value | Description | +| --- | --- | +| `NodeInternalIP` | NodeInternalIP detects a node IP using the first status.Addresses entry of the relevant IP family with type NodeInternalIP on the Kubernetes nodes API. | + + +### L7LogCollectorDaemonSet + + + +L7LogCollectorDaemonSet is the configuration for the L7LogCollector DaemonSet. + +_Appears in:_ +- [ApplicationLayerSpec](#applicationlayerspec) + +| Field | Description | +| --- | --- | +| `spec` _[L7LogCollectorDaemonSetSpec](#l7logcollectordaemonsetspec)_ | (Optional) Spec is the specification of the L7LogCollector DaemonSet. | + + +### L7LogCollectorDaemonSetContainer + + + +L7LogCollectorDaemonSetContainer is a L7LogCollector DaemonSet container. + +_Appears in:_ +- [L7LogCollectorDaemonSetPodSpec](#l7logcollectordaemonsetpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the L7LogCollector DaemonSet container by name.
Supported values are: l7-collector, envoy-proxy, dikastes | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named L7LogCollector DaemonSet container's resources. If omitted, the L7LogCollector DaemonSet will use its default value for this container's resources. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### L7LogCollectorDaemonSetInitContainer + + + +L7LogCollectorDaemonSetInitContainer is a L7LogCollector DaemonSet init container. + +_Appears in:_ +- [L7LogCollectorDaemonSetPodSpec](#l7logcollectordaemonsetpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the L7LogCollector DaemonSet init container by name. | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named L7LogCollector DaemonSet init container's resources. If omitted, the L7LogCollector DaemonSet will use its default value for this init container's resources. | + + +### L7LogCollectorDaemonSetPodSpec + + + +L7LogCollectorDaemonSetPodSpec is the L7LogCollector DaemonSet's PodSpec. + +_Appears in:_ +- [L7LogCollectorDaemonSetPodTemplateSpec](#l7logcollectordaemonsetpodtemplatespec) + +| Field | Description | +| --- | --- | +| `initContainers` _[L7LogCollectorDaemonSetInitContainer](#l7logcollectordaemonsetinitcontainer) array_ | (Optional) InitContainers is a list of L7LogCollector DaemonSet init containers. If specified, this overrides the specified L7LogCollector DaemonSet init containers. If omitted, the L7LogCollector DaemonSet will use its default values for its init containers. | +| `containers` _[L7LogCollectorDaemonSetContainer](#l7logcollectordaemonsetcontainer) array_ | (Optional) Containers is a list of L7LogCollector DaemonSet containers. If specified, this overrides the specified L7LogCollector DaemonSet containers. If omitted, the L7LogCollector DaemonSet will use its default values for its containers. | + + +### L7LogCollectorDaemonSetPodTemplateSpec + + + +L7LogCollectorDaemonSetPodTemplateSpec is the L7LogCollector DaemonSet's PodTemplateSpec + +_Appears in:_ +- [L7LogCollectorDaemonSetSpec](#l7logcollectordaemonsetspec) + +| Field | Description | +| --- | --- | +| `spec` _[L7LogCollectorDaemonSetPodSpec](#l7logcollectordaemonsetpodspec)_ | (Optional) Spec is the L7LogCollector DaemonSet's PodSpec. | + + +### L7LogCollectorDaemonSetSpec + + + +L7LogCollectorDaemonSetSpec defines configuration for the L7LogCollector DaemonSet. + +_Appears in:_ +- [L7LogCollectorDaemonSet](#l7logcollectordaemonset) + +| Field | Description | +| --- | --- | +| `template` _[L7LogCollectorDaemonSetPodTemplateSpec](#l7logcollectordaemonsetpodtemplatespec)_ | (Optional) Template describes the L7LogCollector DaemonSet pod that will be created. | + + +### LinuxDataplaneOption + +_Underlying type:_ _string_ + +LinuxDataplaneOption controls which dataplane is to be used on Linux nodes. + +One of: Iptables, BPF, VPP, Nftables + +_Validation:_ +- Enum: [Iptables BPF VPP Nftables] + + +_Appears in:_ +- [CalicoNetworkSpec](#caliconetworkspec) + +| Value | Description | +| --- | --- | +| `Iptables` | | +| `BPF` | | +| `VPP` | | +| `Nftables` | | + + +### LinuxPodInterfaceType + +_Underlying type:_ _string_ + +LinuxPodInterfaceType specifies the type of virtual device the Calico CNI plugin +creates for the pod-side interface on Linux nodes. + +One of: Veth, Netkit + +_Validation:_ +- Enum: [Veth Netkit] + + +_Appears in:_ +- [CalicoNetworkSpec](#caliconetworkspec) + +| Value | Description | +| --- | --- | +| `Veth` | | +| `Netkit` | | + + +### LogCollectionSpec + + + + + +_Appears in:_ +- [ApplicationLayerSpec](#applicationlayerspec) + +| Field | Description | +| --- | --- | +| `collectLogs` _[LogCollectionStatusType](#logcollectionstatustype)_ | (Optional) This setting enables or disable log collection. Allowed values are Enabled or Disabled. | +| `logIntervalSeconds` _integer_ | (Optional) Interval in seconds for sending L7 log information for processing.
Default: 5 sec | +| `logRequestsPerInterval` _integer_ | (Optional) Maximum number of unique L7 logs that are sent LogIntervalSeconds. Adjust this to limit the number of L7 logs sent per LogIntervalSeconds to felix for further processing, use negative number to ignore limits.
Default: -1 | + + +### LogCollectionStatusType + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [Enabled Disabled] + + +_Appears in:_ +- [IstioSpec](#istiospec) +- [LogCollectionSpec](#logcollectionspec) + +| Value | Description | +| --- | --- | +| `Disabled` | | +| `Enabled` | | + + +### LogCollector + + + +LogCollector installs the components required for Tigera flow and DNS log collection. At most one instance +of this resource is supported. It must be named "tigera-secure". When created, this installs fluentd on all nodes +configured to collect Tigera log data and export it to Tigera's Elasticsearch cluster as well as any additionally configured destinations. + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `LogCollector` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[LogCollectorSpec](#logcollectorspec)_ | Specification of the desired state for Tigera log collection. | +| `status` _[LogCollectorStatus](#logcollectorstatus)_ | Most recently observed state for Tigera log collection. | + + +### LogCollectorSpec + + + +LogCollectorSpec defines the desired state of Tigera flow, audit, and DNS log collection. + +_Appears in:_ +- [LogCollector](#logcollector) + +| Field | Description | +| --- | --- | +| `additionalStores` _[AdditionalLogStoreSpec](#additionallogstorespec)_ | (Optional) Configuration for exporting flow, audit, and DNS logs to external storage. | +| `additionalSources` _[AdditionalLogSourceSpec](#additionallogsourcespec)_ | (Optional) Configuration for importing audit logs from managed kubernetes cluster log sources. | +| `collectProcessPath` _[CollectProcessPathOption](#collectprocesspathoption)_ | (Optional) Configuration for enabling/disabling process path collection in flowlogs. If Enabled, this feature sets hostPID to true in order to read process cmdline.
Default: Enabled | +| `multiTenantManagementClusterNamespace` _string_ | (Optional) If running as a multi-tenant management cluster, the namespace in which the management cluster's tenant services are running. | +| `fluentdDaemonSet` _[FluentdDaemonSet](#fluentddaemonset)_ | FluentdDaemonSet configures the Fluentd DaemonSet. | +| `eksLogForwarderDeployment` _[EKSLogForwarderDeployment](#ekslogforwarderdeployment)_ | (Optional) EKSLogForwarderDeployment configures the EKSLogForwarderDeployment Deployment. | + + +### LogCollectorStatus + + + +LogCollectorStatus defines the observed state of Tigera flow and DNS log collection + +_Appears in:_ +- [LogCollector](#logcollector) + +| Field | Description | +| --- | --- | +| `state` _string_ | State provides user-readable status. | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#condition-v1-meta) array_ | (Optional) Conditions represents the latest observed set of conditions for the component. A component may be one or more of Ready, Progressing, Degraded or other customer types. | + + +### LogLevel + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [Error Warning Info Debug] + + +_Appears in:_ +- [CNILogging](#cnilogging) + +| Value | Description | +| --- | --- | +| `Error` | | +| `Warn` | | +| `Info` | | +| `Debug` | | + + +### LogSeverity + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [Fatal Error Warn Info Debug Trace] + + +_Appears in:_ +- [APIServerLogging](#apiserverlogging) +- [EgressGatewaySpec](#egressgatewayspec) +- [QueryServerLogging](#queryserverlogging) + +| Value | Description | +| --- | --- | +| `Fatal` | | +| `Error` | | +| `Warn` | | +| `Info` | | +| `Debug` | | +| `Trace` | | + + +### Logging + + + + + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `cni` _[CNILogging](#cnilogging)_ | (Optional) Customized logging specification for calico-cni plugin | + + +### ManagementClusterConnection + + + +ManagementClusterConnection represents a link between a managed cluster and a management cluster. At most one +instance of this resource is supported. It must be named "tigera-secure". + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `ManagementClusterConnection` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[ManagementClusterConnectionSpec](#managementclusterconnectionspec)_ | | +| `status` _[ManagementClusterConnectionStatus](#managementclusterconnectionstatus)_ | | + + +### ManagementClusterConnectionSpec + + + +ManagementClusterConnectionSpec defines the desired state of ManagementClusterConnection + +_Appears in:_ +- [ManagementClusterConnection](#managementclusterconnection) + +| Field | Description | +| --- | --- | +| `managementClusterAddr` _string_ | (Optional) Specify where the managed cluster can reach the management cluster. Ex.: "10.128.0.10:30449". A managed cluster should be able to access this address. This field is used by managed clusters only. | +| `tls` _[ManagementClusterTLS](#managementclustertls)_ | (Optional) TLS provides options for configuring how Managed Clusters can establish an mTLS connection with the Management Cluster. | +| `guardianDeployment` _[GuardianDeployment](#guardiandeployment)_ | GuardianDeployment configures the guardian Deployment. | +| `impersonation` _[Impersonation](#impersonation)_ | (Optional) Impersonation configures the RBAC impersonation permissions for the guardian deployment. This field is not applicable to installation variant Calico as no impersonation is ever used. Otherwise, if this field is left nil, a default set of permissions will be applied.
WARNING: If this field is specified, it completely replaces the default permissions. For example, providing an empty `impersonation: \{\}` block will result in guardian having NO impersonation permissions. Similarly, if you specify `users` but omit `groups`, guardian will lose its default permissions to impersonate groups. | + + +### ManagementClusterConnectionStatus + + + +ManagementClusterConnectionStatus defines the observed state of ManagementClusterConnection + +_Appears in:_ +- [ManagementClusterConnection](#managementclusterconnection) + +| Field | Description | +| --- | --- | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#condition-v1-meta) array_ | (Optional) Conditions represents the latest observed set of conditions for the component. A component may be one or more of Ready, Progressing, Degraded or other customer types. | + + +### ManagementClusterTLS + + + + + +_Appears in:_ +- [ManagementClusterConnectionSpec](#managementclusterconnectionspec) + +| Field | Description | +| --- | --- | +| `ca` _[CAType](#catype)_ | CA indicates which verification method the tunnel client should use to verify the tunnel server's identity. When left blank or set to 'Tigera', the tunnel client will expect a self-signed cert to be included in the certificate bundle and will expect the cert to have a Common Name (CN) of 'voltron'. When set to 'Public', the tunnel client will use its installed system certs and will use the managementClusterAddr to verify the tunnel server's identity.
Default: Tigera | + + +### Metadata + + + +Metadata contains the standard Kubernetes labels and annotations fields. + +_Appears in:_ +- [APIServerDeployment](#apiserverdeployment) +- [APIServerDeploymentPodTemplateSpec](#apiserverdeploymentpodtemplatespec) +- [CSINodeDriverDaemonSet](#csinodedriverdaemonset) +- [CSINodeDriverDaemonSetPodTemplateSpec](#csinodedriverdaemonsetpodtemplatespec) +- [CalicoKubeControllersDeployment](#calicokubecontrollersdeployment) +- [CalicoKubeControllersDeploymentPodTemplateSpec](#calicokubecontrollersdeploymentpodtemplatespec) +- [CalicoNodeDaemonSet](#caliconodedaemonset) +- [CalicoNodeDaemonSetPodTemplateSpec](#caliconodedaemonsetpodtemplatespec) +- [CalicoNodeWindowsDaemonSet](#caliconodewindowsdaemonset) +- [CalicoNodeWindowsDaemonSetPodTemplateSpec](#caliconodewindowsdaemonsetpodtemplatespec) +- [CalicoWebhooksDeployment](#calicowebhooksdeployment) +- [CalicoWebhooksDeploymentPodTemplateSpec](#calicowebhooksdeploymentpodtemplatespec) +- [CalicoWindowsUpgradeDaemonSet](#calicowindowsupgradedaemonset) +- [CalicoWindowsUpgradeDaemonSetPodTemplateSpec](#calicowindowsupgradedaemonsetpodtemplatespec) +- [GatewayCertgenJob](#gatewaycertgenjob) +- [GatewayCertgenJobPodTemplate](#gatewaycertgenjobpodtemplate) +- [GatewayControllerDeployment](#gatewaycontrollerdeployment) +- [GatewayControllerDeploymentPodTemplate](#gatewaycontrollerdeploymentpodtemplate) +- [GatewayDaemonSetPodTemplate](#gatewaydaemonsetpodtemplate) +- [GatewayDeploymentPodTemplate](#gatewaydeploymentpodtemplate) +- [GatewayService](#gatewayservice) +- [GoldmaneDeployment](#goldmanedeployment) +- [GoldmaneDeploymentPodTemplateSpec](#goldmanedeploymentpodtemplatespec) +- [PodDisruptionBudgetOverride](#poddisruptionbudgetoverride) +- [TyphaDeployment](#typhadeployment) +- [TyphaDeploymentPodTemplateSpec](#typhadeploymentpodtemplatespec) +- [WhiskerDeployment](#whiskerdeployment) +- [WhiskerDeploymentPodTemplateSpec](#whiskerdeploymentpodtemplatespec) + +| Field | Description | +| --- | --- | +| `labels` _object (keys:string, values:string)_ | (Optional) Labels is a map of string keys and values that may match replicaset and service selectors. Each of these key/value pairs are added to the object's labels provided the key does not already exist in the object's labels. | +| `annotations` _object (keys:string, values:string)_ | (Optional) Annotations is a map of arbitrary non-identifying metadata. Each of these key/value pairs are added to the object's annotations provided the key does not already exist in the object's annotations. | + + +### Monitor + + + +Monitor is the Schema for the monitor API. At most one instance +of this resource is supported. It must be named "tigera-secure". + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `Monitor` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[MonitorSpec](#monitorspec)_ | | +| `status` _[MonitorStatus](#monitorstatus)_ | | + + +### MonitorSpec + + + +MonitorSpec defines the desired state of Tigera monitor. + +_Appears in:_ +- [Monitor](#monitor) + +| Field | Description | +| --- | --- | +| `externalPrometheus` _[ExternalPrometheus](#externalprometheus)_ | ExternalPrometheus optionally configures integration with an external Prometheus for scraping Calico metrics. When specified, the operator will render resources in the defined namespace. This option can be useful for configuring scraping from git-ops tools without the need of post-installation steps. | +| `prometheus` _[Prometheus](#prometheus)_ | (Optional) Prometheus is the configuration for the Prometheus. | +| `alertManager` _[Alertmanager](#alertmanager)_ | (Optional) Alertmanager is the configuration for the Alertmanager. | + + +### MonitorStatus + + + +MonitorStatus defines the observed state of Tigera monitor. + +_Appears in:_ +- [Monitor](#monitor) + +| Field | Description | +| --- | --- | +| `state` _string_ | State provides user-readable status. | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#condition-v1-meta) array_ | (Optional) Conditions represents the latest observed set of conditions for the component. A component may be one or more of Ready, Progressing, Degraded or other customer types. | + + +### MultiInterfaceMode + +_Underlying type:_ _string_ + +MultiInterfaceMode describes the method of providing multiple pod interfaces. + +One of: None, Multus + +_Appears in:_ +- [CalicoNetworkSpec](#caliconetworkspec) + +| Value | Description | +| --- | --- | +| `None` | | +| `Multus` | | + + +### NATOutgoingType + +_Underlying type:_ _string_ + +NATOutgoingType describe the type of outgoing NAT to use. + +One of: Enabled, Disabled + +_Appears in:_ +- [IPPool](#ippool) + +| Value | Description | +| --- | --- | +| `Enabled` | | +| `Disabled` | | + + +### NamespacedName + + + +NamespacedName references an object of a known type in any namespace. + +_Appears in:_ +- [GatewayAPISpec](#gatewayapispec) +- [GatewayClassSpec](#gatewayclassspec) + +| Field | Description | +| --- | --- | +| `namespace` _string_ | | +| `name` _string_ | | + + +### NativeIP + +_Underlying type:_ _string_ + +NativeIP defines if Egress Gateway pods should have AWS IPs. +When NativeIP is enabled, the IPPools should be backed by AWS subnet. + +_Appears in:_ +- [AWSEgressGateway](#awsegressgateway) + +| Value | Description | +| --- | --- | +| `Enabled` | | +| `Disabled` | | + + +### NetworkPolicyManagement + +_Underlying type:_ _string_ + +NetworkPolicyManagement specifies whether the operator manages the NetworkPolicies it installs to +protect the Calico components it manages. + +_Validation:_ +- Enum: [Enabled Disabled] + + +_Appears in:_ +- [NetworkPolicySpec](#networkpolicyspec) + +| Value | Description | +| --- | --- | +| `Enabled` | | +| `Disabled` | | + + +### NetworkPolicySpec + + + +NetworkPolicySpec configures how the operator manages the NetworkPolicies it installs. + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `managePolicies` _[NetworkPolicyManagement](#networkpolicymanagement)_ | (Optional) ManagePolicies controls whether the operator creates and reconciles the NetworkPolicies and GlobalNetworkPolicies it uses to protect the Calico components it installs. When set to Disabled, the operator stops creating or updating these policies and deletes any it has already created, leaving policy management to the user. Defaults to Enabled. | + + +### NodeAddressAutodetection + + + +NodeAddressAutodetection provides configuration options for auto-detecting node addresses. At most one option +can be used. If no detection option is specified, then IP auto detection will be disabled for this address family and IPs +must be specified directly on the Node resource. + +_Appears in:_ +- [CalicoNetworkSpec](#caliconetworkspec) + +| Field | Description | +| --- | --- | +| `firstFound` _boolean_ | (Optional) FirstFound uses default interface matching parameters to select an interface, performing best-effort filtering based on well-known interface names. | +| `kubernetes` _[KubernetesAutodetectionMethod](#kubernetesautodetectionmethod)_ | (Optional) Kubernetes configures Calico to detect node addresses based on the Kubernetes API. | +| `interface` _string_ | (Optional) Interface enables IP auto-detection based on interfaces that match the given regex. | +| `skipInterface` _string_ | (Optional) SkipInterface enables IP auto-detection based on interfaces that do not match the given regex. | +| `canReach` _string_ | (Optional) CanReach enables IP auto-detection based on which source address on the node is used to reach the specified IP or domain. | +| `cidrs` _string array_ | CIDRS enables IP auto-detection based on which addresses on the nodes are within one of the provided CIDRs. | + + +### NodeAffinity + + + +NodeAffinity is similar to *v1.NodeAffinity, but allows us to limit available schedulers. + +_Appears in:_ +- [TyphaAffinity](#typhaaffinity) + +| Field | Description | +| --- | --- | +| `preferredDuringSchedulingIgnoredDuringExecution` _[PreferredSchedulingTerm](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#preferredschedulingterm-v1-core) array_ | (Optional) The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified by this field, but it may choose a node that violates one or more of the expressions. | +| `requiredDuringSchedulingIgnoredDuringExecution` _[NodeSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#nodeselector-v1-core)_ | (Optional)
WARNING: Please note that if the affinity requirements specified by this field are not met at scheduling time, the pod will NOT be scheduled onto the node. There is no fallback to another affinity rules with this setting. This may cause networking disruption or even catastrophic failure! PreferredDuringSchedulingIgnoredDuringExecution should be used for affinity unless there is a specific well understood reason to use RequiredDuringSchedulingIgnoredDuringExecution and you can guarantee that the RequiredDuringSchedulingIgnoredDuringExecution will always have sufficient nodes to satisfy the requirement. NOTE: RequiredDuringSchedulingIgnoredDuringExecution is set by default for AKS nodes, to avoid scheduling Typhas on virtual-nodes. If the affinity requirements specified by this field cease to be met at some point during pod execution (e.g. due to an update), the system may or may not try to eventually evict the pod from its node. | + + +### NonClusterHost + + + +NonClusterHost installs the components required for non-cluster host log collection. +At most one instance of this resource is supported. It must be named "tigera-secure". + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `NonClusterHost` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[NonClusterHostSpec](#nonclusterhostspec)_ | Specification of the desired state for non-cluster host log collection. | + + +### NonClusterHostSpec + + + +NonClusterHostSpec enables non-cluster hosts to connect to a cluster. + +_Appears in:_ +- [NonClusterHost](#nonclusterhost) + +| Field | Description | +| --- | --- | +| `endpoint` _string_ | Location of the log ingestion point for non-cluster hosts. For example: https://1.2.3.4:443 | +| `typhaEndpoint` _string_ | Location of the Typha endpoint for non-cluster host Felix and Typha communication. For example: 5.6.7.8:5473 | + + +### NonPrivilegedType + +_Underlying type:_ _string_ + +NonPrivilegedType specifies whether Calico runs as permissioned or not + +One of: Enabled, Disabled + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Value | Description | +| --- | --- | +| `Enabled` | | +| `Disabled` | | + + +### NotificationMode + +_Underlying type:_ _string_ + + + +_Appears in:_ +- [WhiskerSpec](#whiskerspec) + +| Value | Description | +| --- | --- | +| `Disabled` | | +| `Enabled` | | + + +### PacketCaptureAPI + + + +PacketCaptureAPI is used to configure the resource requirement for PacketCaptureAPI deployment. It must be named "tigera-secure". + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `PacketCaptureAPI` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[PacketCaptureAPISpec](#packetcaptureapispec)_ | Specification of the desired state for the PacketCaptureAPI. | +| `status` _[PacketCaptureAPIStatus](#packetcaptureapistatus)_ | Most recently observed state for the PacketCaptureAPI. | + + +### PacketCaptureAPIDeployment + + + +PacketCaptureAPIDeployment is the configuration for the PacketCaptureAPI Deployment. + +_Appears in:_ +- [PacketCaptureAPISpec](#packetcaptureapispec) + +| Field | Description | +| --- | --- | +| `spec` _[PacketCaptureAPIDeploymentSpec](#packetcaptureapideploymentspec)_ | (Optional) Spec is the specification of the PacketCaptureAPI Deployment. | + + +### PacketCaptureAPIDeploymentContainer + + + +PacketCaptureAPIDeploymentContainer is a PacketCaptureAPI Deployment container. + +_Appears in:_ +- [PacketCaptureAPIDeploymentPodSpec](#packetcaptureapideploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the PacketCaptureAPI Deployment container by name.
Supported values are: tigera-packetcapture-server | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named PacketCaptureAPI Deployment container's resources. If omitted, the PacketCaptureAPI Deployment will use its default value for this container's resources. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### PacketCaptureAPIDeploymentInitContainer + + + +PacketCaptureAPIDeploymentInitContainer is a PacketCaptureAPI Deployment init container. + +_Appears in:_ +- [PacketCaptureAPIDeploymentPodSpec](#packetcaptureapideploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the PacketCaptureAPI Deployment init container by name.
Supported values are: tigera-packetcapture-server-tls-key-cert-provisioner | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named PacketCaptureAPI Deployment init container's resources. If omitted, the PacketCaptureAPI Deployment will use its default value for this init container's resources. | + + +### PacketCaptureAPIDeploymentPodSpec + + + +PacketCaptureAPIDeploymentPodSpec is the PacketCaptureAPI Deployment's PodSpec. + +_Appears in:_ +- [PacketCaptureAPIDeploymentPodTemplateSpec](#packetcaptureapideploymentpodtemplatespec) + +| Field | Description | +| --- | --- | +| `initContainers` _[PacketCaptureAPIDeploymentInitContainer](#packetcaptureapideploymentinitcontainer) array_ | (Optional) InitContainers is a list of PacketCaptureAPI init containers. If specified, this overrides the specified PacketCaptureAPI Deployment init containers. If omitted, the PacketCaptureAPI Deployment will use its default values for its init containers. | +| `containers` _[PacketCaptureAPIDeploymentContainer](#packetcaptureapideploymentcontainer) array_ | (Optional) Containers is a list of PacketCaptureAPI containers. If specified, this overrides the specified PacketCaptureAPI Deployment containers. If omitted, the PacketCaptureAPI Deployment will use its default values for its containers. | + + +### PacketCaptureAPIDeploymentPodTemplateSpec + + + +PacketCaptureAPIDeploymentPodTemplateSpec is the PacketCaptureAPI Deployment's PodTemplateSpec + +_Appears in:_ +- [PacketCaptureAPIDeploymentSpec](#packetcaptureapideploymentspec) + +| Field | Description | +| --- | --- | +| `spec` _[PacketCaptureAPIDeploymentPodSpec](#packetcaptureapideploymentpodspec)_ | (Optional) Spec is the PacketCaptureAPI Deployment's PodSpec. | + + +### PacketCaptureAPIDeploymentSpec + + + +PacketCaptureAPIDeploymentSpec defines configuration for the PacketCaptureAPI Deployment. + +_Appears in:_ +- [PacketCaptureAPIDeployment](#packetcaptureapideployment) + +| Field | Description | +| --- | --- | +| `template` _[PacketCaptureAPIDeploymentPodTemplateSpec](#packetcaptureapideploymentpodtemplatespec)_ | (Optional) Template describes the PacketCaptureAPI Deployment pod that will be created. | + + +### PacketCaptureAPISpec + + + +PacketCaptureAPISpec defines configuration for the Packet Capture API. + +_Appears in:_ +- [PacketCaptureAPI](#packetcaptureapi) + +| Field | Description | +| --- | --- | +| `packetCaptureAPIDeployment` _[PacketCaptureAPIDeployment](#packetcaptureapideployment)_ | (Optional) PacketCaptureAPIDeployment configures the PacketCaptureAPI Deployment. | + + +### PacketCaptureAPIStatus + + + +PacketCaptureAPIStatus defines the observed state of the Packet Capture API. + +_Appears in:_ +- [PacketCaptureAPI](#packetcaptureapi) + +| Field | Description | +| --- | --- | +| `state` _string_ | State provides user-readable status. | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#condition-v1-meta) array_ | (Optional) Conditions represents the latest observed set of conditions for the component. A component may be one or more of Ready, Progressing, Degraded or other customer types. | + + +### PathMatch + + + + + +_Appears in:_ +- [TLSTerminatedRouteSpec](#tlsterminatedroutespec) + +| Field | Description | +| --- | --- | +| `path` _string_ | Path is the path portion of the URL based on which we proxy. | +| `pathRegexp` _string_ | (Optional) PathRegexp, if not nil, checks if Regexp matches the path. | +| `pathReplace` _string_ | (Optional) PathReplace if not nil will be used to replace PathRegexp matches. | + + +### PodDisruptionBudgetOverride + + + +PodDisruptionBudgetOverride allows overriding select fields on an operator-managed +PodDisruptionBudget. The PDB's selector, name, and namespace are managed by the +operator and cannot be overridden. + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[PodDisruptionBudgetOverrideSpec](#poddisruptionbudgetoverridespec)_ | (Optional) Spec is the specification of the PodDisruptionBudget. | + + +### PodDisruptionBudgetOverrideSpec + + + +PodDisruptionBudgetOverrideSpec defines the desired state of an operator-managed PodDisruptionBudget. + +_Appears in:_ +- [PodDisruptionBudgetOverride](#poddisruptionbudgetoverride) + +| Field | Description | +| --- | --- | +| `minAvailable` _[IntOrString](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#intorstring-intstr-util)_ | (Optional) MinAvailable is the minimum number of pods (as an integer or percentage) that must remain available during a disruption. Mutually exclusive with MaxUnavailable. | +| `maxUnavailable` _[IntOrString](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#intorstring-intstr-util)_ | (Optional) MaxUnavailable is the maximum number of pods (as an integer or percentage) that can be unavailable during a disruption. Mutually exclusive with MinAvailable. If neither MinAvailable nor MaxUnavailable is set, the operator applies its default (MaxUnavailable=1 for calico-typha). | +| `unhealthyPodEvictionPolicy` _[UnhealthyPodEvictionPolicyType](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#unhealthypodevictionpolicytype-v1-policy)_ | (Optional) UnhealthyPodEvictionPolicy defines when unhealthy pods should be considered for eviction. Defaults to IfHealthyBudget (the Kubernetes default) when unset. See https://kubernetes.io/docs/tasks/run-application/configure-pdb/#unhealthy-pod-eviction-policy. | + + +### PolicyMode + +_Underlying type:_ _string_ + + + +_Appears in:_ +- [Azure](#azure) + +| Value | Description | +| --- | --- | +| `Default` | | +| `Manual` | | + + +### PolicyRecommendation + + + +PolicyRecommendation is the Schema for the policy recommendation API. At most one instance +of this resource is supported. It must be named "tigera-secure". + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `PolicyRecommendation` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[PolicyRecommendationSpec](#policyrecommendationspec)_ | | +| `status` _[PolicyRecommendationStatus](#policyrecommendationstatus)_ | | + + +### PolicyRecommendationDeployment + + + +PolicyRecommendationDeployment is the configuration for the PolicyRecommendation Deployment. + +_Appears in:_ +- [PolicyRecommendationSpec](#policyrecommendationspec) + +| Field | Description | +| --- | --- | +| `spec` _[PolicyRecommendationDeploymentSpec](#policyrecommendationdeploymentspec)_ | (Optional) Spec is the specification of the PolicyRecommendation Deployment. | + + +### PolicyRecommendationDeploymentContainer + + + +PolicyRecommendationDeploymentContainer is a PolicyRecommendation Deployment container. + +_Appears in:_ +- [PolicyRecommendationDeploymentPodSpec](#policyrecommendationdeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the PolicyRecommendation Deployment container by name.
Supported values are: policy-recommendation-controller | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named PolicyRecommendation Deployment container's resources. If omitted, the PolicyRecommendation Deployment will use its default value for this container's resources. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### PolicyRecommendationDeploymentInitContainer + + + +PolicyRecommendationDeploymentInitContainer is a PolicyRecommendation Deployment init container. + +_Appears in:_ +- [PolicyRecommendationDeploymentPodSpec](#policyrecommendationdeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the PolicyRecommendation Deployment init container by name. | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named PolicyRecommendation Deployment init container's resources. If omitted, the PolicyRecommendation Deployment will use its default value for this init container's resources. | + + +### PolicyRecommendationDeploymentPodSpec + + + +PolicyRecommendationDeploymentPodSpec is the PolicyRecommendation Deployment's PodSpec. + +_Appears in:_ +- [PolicyRecommendationDeploymentPodTemplateSpec](#policyrecommendationdeploymentpodtemplatespec) + +| Field | Description | +| --- | --- | +| `initContainers` _[PolicyRecommendationDeploymentInitContainer](#policyrecommendationdeploymentinitcontainer) array_ | (Optional) InitContainers is a list of PolicyRecommendation init containers. If specified, this overrides the specified PolicyRecommendation Deployment init containers. If omitted, the PolicyRecommendation Deployment will use its default values for its init containers. | +| `containers` _[PolicyRecommendationDeploymentContainer](#policyrecommendationdeploymentcontainer) array_ | (Optional) Containers is a list of PolicyRecommendation containers. If specified, this overrides the specified PolicyRecommendation Deployment containers. If omitted, the PolicyRecommendation Deployment will use its default values for its containers. | + + +### PolicyRecommendationDeploymentPodTemplateSpec + + + +PolicyRecommendationDeploymentPodTemplateSpec is the PolicyRecommendation Deployment's PodTemplateSpec + +_Appears in:_ +- [PolicyRecommendationDeploymentSpec](#policyrecommendationdeploymentspec) + +| Field | Description | +| --- | --- | +| `spec` _[PolicyRecommendationDeploymentPodSpec](#policyrecommendationdeploymentpodspec)_ | (Optional) Spec is the PolicyRecommendation Deployment's PodSpec. | + + +### PolicyRecommendationDeploymentSpec + + + +PolicyRecommendationDeploymentSpec defines configuration for the PolicyRecommendation Deployment. + +_Appears in:_ +- [PolicyRecommendationDeployment](#policyrecommendationdeployment) + +| Field | Description | +| --- | --- | +| `template` _[PolicyRecommendationDeploymentPodTemplateSpec](#policyrecommendationdeploymentpodtemplatespec)_ | (Optional) Template describes the PolicyRecommendation Deployment pod that will be created. | + + +### PolicyRecommendationSpec + + + +PolicyRecommendationSpec defines configuration for the Calico Enterprise Policy Recommendation +service. + +_Appears in:_ +- [PolicyRecommendation](#policyrecommendation) + +| Field | Description | +| --- | --- | +| `policyRecommendationDeployment` _[PolicyRecommendationDeployment](#policyrecommendationdeployment)_ | (Optional) PolicyRecommendation configures the PolicyRecommendation Deployment. | + + +### PolicyRecommendationStatus + + + +PolicyRecommendationStatus defines the observed state of Tigera policy recommendation. + +_Appears in:_ +- [PolicyRecommendation](#policyrecommendation) + +| Field | Description | +| --- | --- | +| `state` _string_ | State provides user-readable status. | + + +### ProbeOverride + + + +ProbeOverride allows customization of probe timing parameters without +changing the probe handler itself (which the operator controls). + +_Appears in:_ +- [APIServerDeploymentContainer](#apiserverdeploymentcontainer) +- [CSINodeDriverDaemonSetContainer](#csinodedriverdaemonsetcontainer) +- [CalicoKubeControllersDeploymentContainer](#calicokubecontrollersdeploymentcontainer) +- [CalicoNodeDaemonSetContainer](#caliconodedaemonsetcontainer) +- [CalicoNodeWindowsDaemonSetContainer](#caliconodewindowsdaemonsetcontainer) +- [CalicoWebhooksDeploymentContainer](#calicowebhooksdeploymentcontainer) +- [CalicoWindowsUpgradeDaemonSetContainer](#calicowindowsupgradedaemonsetcontainer) +- [EGWDeploymentContainer](#egwdeploymentcontainer) +- [EKSLogForwarderDeploymentContainer](#ekslogforwarderdeploymentcontainer) +- [FluentdDaemonSetContainer](#fluentddaemonsetcontainer) +- [GatewayCertgenJobContainer](#gatewaycertgenjobcontainer) +- [GatewayControllerDeploymentContainer](#gatewaycontrollerdeploymentcontainer) +- [GatewayDaemonSetContainer](#gatewaydaemonsetcontainer) +- [GatewayDeploymentContainer](#gatewaydeploymentcontainer) +- [GoldmaneDeploymentContainer](#goldmanedeploymentcontainer) +- [GuardianDeploymentContainer](#guardiandeploymentcontainer) +- [IntrusionDetectionControllerDeploymentContainer](#intrusiondetectioncontrollerdeploymentcontainer) +- [L7LogCollectorDaemonSetContainer](#l7logcollectordaemonsetcontainer) +- [PacketCaptureAPIDeploymentContainer](#packetcaptureapideploymentcontainer) +- [PolicyRecommendationDeploymentContainer](#policyrecommendationdeploymentcontainer) +- [PrometheusContainer](#prometheuscontainer) +- [TyphaDeploymentContainer](#typhadeploymentcontainer) +- [WhiskerDeploymentContainer](#whiskerdeploymentcontainer) + +| Field | Description | +| --- | --- | +| `periodSeconds` _integer_ | (Optional) PeriodSeconds is how often (in seconds) to perform the probe. | +| `timeoutSeconds` _integer_ | (Optional) TimeoutSeconds is the number of seconds after which the probe times out. | +| `failureThreshold` _integer_ | (Optional) FailureThreshold is the minimum consecutive failures for the probe to be considered failed after having succeeded. | +| `initialDelaySeconds` _integer_ | (Optional) InitialDelaySeconds is the number of seconds after the container starts before the probe is initiated. | + + +### ProductVariant + +_Underlying type:_ _string_ + +ProductVariant represents the variant of the product. + +One of: Calico, CalicoEnterprise. +TigeraSecureEnterprise is a deprecated alias for CalicoEnterprise. + +_Appears in:_ +- [InstallationSpec](#installationspec) +- [InstallationStatus](#installationstatus) + + + +### Prometheus + + + + + +_Appears in:_ +- [MonitorSpec](#monitorspec) + +| Field | Description | +| --- | --- | +| `spec` _[PrometheusSpec](#prometheusspec)_ | (Optional) Spec is the specification of the Prometheus. | + + +### PrometheusContainer + + + +PrometheusContainer is a Prometheus container. + +_Appears in:_ +- [CommonPrometheusFields](#commonprometheusfields) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the Prometheus Deployment container by name.
Supported values are: authn-proxy | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named Prometheus container's resources. If omitted, the Prometheus will use its default value for this container's resources. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### PrometheusSpec + + + + + +_Appears in:_ +- [Prometheus](#prometheus) + +| Field | Description | +| --- | --- | +| `commonPrometheusFields` _[CommonPrometheusFields](#commonprometheusfields)_ | CommonPrometheusFields are the options available to both the Prometheus server and agent. | + + +### Provider + +_Underlying type:_ _string_ + +Provider represents a particular provider or flavor of Kubernetes. Valid options +are: EKS, GKE, AKS, RKE2, OpenShift, DockerEnterprise, TKG, Kind. + +_Appears in:_ +- [InstallationSpec](#installationspec) + + + +### Proxy + + + + + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `httpProxy` _string_ | (Optional) HTTPProxy defines the value of the HTTP_PROXY environment variable that will be set on Tigera containers that connect to destinations outside the cluster. | +| `httpsProxy` _string_ | (Optional) HTTPSProxy defines the value of the HTTPS_PROXY environment variable that will be set on Tigera containers that connect to destinations outside the cluster. | +| `noProxy` _string_ | (Optional) NoProxy defines the value of the NO_PROXY environment variable that will be set on Tigera containers that connect to destinations outside the cluster. This value must be set such that destinations within the scope of the cluster, including the Kubernetes API server, are exempt from being proxied. | + + +### QueryServerLogging + + + + + +_Appears in:_ +- [APIServerPodLogging](#apiserverpodlogging) + +| Field | Description | +| --- | --- | +| `logSeverity` _[LogSeverity](#logseverity)_ | (Optional) LogSeverity defines log level for QueryServer container. | + + +### S3StoreSpec + + + +S3StoreSpec defines configuration for exporting logs to Amazon S3. + +_Appears in:_ +- [AdditionalLogStoreSpec](#additionallogstorespec) + +| Field | Description | +| --- | --- | +| `region` _string_ | AWS Region of the S3 bucket | +| `bucketName` _string_ | Name of the S3 bucket to send logs | +| `bucketPath` _string_ | Path in the S3 bucket where to send logs | +| `hostScope` _[HostScope](#hostscope)_ | (Optional) The set of hosts that will forward their logs to this store. | + + +### SNIMatch + + + + + +_Appears in:_ +- [TLSPassThroughRouteSpec](#tlspassthroughroutespec) + +| Field | Description | +| --- | --- | +| `serverName` _string_ | ServerName is used to match the server name for the request. | + + +### ServiceMonitor + + + + + +_Appears in:_ +- [ExternalPrometheus](#externalprometheus) + +| Field | Description | +| --- | --- | +| `labels` _object (keys:string, values:string)_ | Labels are the metadata.labels of the ServiceMonitor. When combined with spec.serviceMonitorSelector.matchLabels on your prometheus instance, the service monitor will automatically be picked up.
Default: k8s-app=tigera-prometheus | +| `endpoints` _[Endpoint](#endpoint) array_ | The endpoints to scrape. This struct contains a subset of the Endpoint as defined in the prometheus docs. Fields related to connecting to our Prometheus server are automatically set by the operator. | + + +### SidecarStatusType + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [Enabled Disabled] + + +_Appears in:_ +- [ApplicationLayerSpec](#applicationlayerspec) + +| Value | Description | +| --- | --- | +| `Enabled` | | +| `Disabled` | | + + +### SidecarWebhookStateType + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [Enabled Disabled] + + +_Appears in:_ +- [ApplicationLayerStatus](#applicationlayerstatus) + +| Value | Description | +| --- | --- | +| `Enabled` | | +| `Disabled` | | + + +### SplunkStoreSpec + + + +SplunkStoreSpec defines configuration for exporting logs to splunk. + +_Appears in:_ +- [AdditionalLogStoreSpec](#additionallogstorespec) + +| Field | Description | +| --- | --- | +| `endpoint` _string_ | Location for splunk's http event collector end point. example `https://1.2.3.4:8088` | +| `hostScope` _[HostScope](#hostscope)_ | (Optional) The set of hosts that will forward their logs to this store | + + +### StatusConditionType + +_Underlying type:_ _string_ + +StatusConditionType is a type of condition that may apply to a particular component. + +_Appears in:_ +- [TigeraStatusCondition](#tigerastatuscondition) + +| Value | Description | +| --- | --- | +| `Available` | Available indicates that the component is healthy. | +| `Progressing` | Progressing means that the component is in the process of being installed or upgraded. | +| `Degraded` | Degraded means the component is not operating as desired and user action is required. | +| `Ready` | Ready indicates that the component is healthy and ready.it is identical to Available and used in Status conditions for CRs. | + + +### Sysctl + + + + + +_Appears in:_ +- [CalicoNetworkSpec](#caliconetworkspec) + +| Field | Description | +| --- | --- | +| `key` _string_ | | +| `value` _string_ | | + + +### SyslogLogType + +_Underlying type:_ _string_ + +SyslogLogType represents the allowable log types for syslog. +Allowable values are Audit, DNS, Flows and IDSEvents. +* Audit corresponds to audit logs for both Kubernetes resources and Enterprise custom resources. +* DNS corresponds to DNS logs generated by Calico node. +* Flows corresponds to flow logs generated by Calico node. +* IDSEvents corresponds to event logs for the intrusion detection system (anomaly detection, suspicious IPs, suspicious domains and global alerts). + +_Validation:_ +- Enum: [Audit DNS Flows IDSEvents] + + +_Appears in:_ +- [SyslogStoreSpec](#syslogstorespec) + +| Value | Description | +| --- | --- | +| `Audit` | | +| `DNS` | | +| `Flows` | | +| `L7` | | +| `IDSEvents` | | + + +### SyslogStoreSpec + + + +SyslogStoreSpec defines configuration for exporting logs to syslog. + +_Appears in:_ +- [AdditionalLogStoreSpec](#additionallogstorespec) + +| Field | Description | +| --- | --- | +| `endpoint` _string_ | Location of the syslog server. example: tcp://1.2.3.4:601 | +| `packetSize` _integer_ | (Optional) PacketSize defines the maximum size of packets to send to syslog. In general this is only needed if you notice long logs being truncated.
Default: 1024 | +| `logTypes` _[SyslogLogType](#sysloglogtype) array_ | If no values are provided, the list will be updated to include log types Audit, DNS and Flows.
Default: Audit, DNS, Flows | +| `encryption` _[EncryptionOption](#encryptionoption)_ | (Optional) Encryption configures traffic encryption to the Syslog server.
Default: None | +| `hostScope` _[HostScope](#hostscope)_ | (Optional) The set of hosts that will forward their logs to this store. | + + +### TLSCipher + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [TLS_AES_256_GCM_SHA384 TLS_CHACHA20_POLY1305_SHA256 TLS_AES_128_GCM_SHA256 TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 TLS_RSA_WITH_AES_256_GCM_SHA384 TLS_RSA_WITH_AES_128_GCM_SHA256 TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA] + + +_Appears in:_ +- [TLSCipherSuite](#tlsciphersuite) + +| Value | Description | +| --- | --- | +| `TLS_AES_256_GCM_SHA384` | TLS 1.3 | +| `TLS_CHACHA20_POLY1305_SHA256` | | +| `TLS_AES_128_GCM_SHA256` | | +| `TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384` | TLS 1.2 | +| `TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384` | | +| `TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256` | | +| `TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256` | | +| `TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256` | | +| `TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256` | | +| `TLS_RSA_WITH_AES_256_GCM_SHA384` | | +| `TLS_RSA_WITH_AES_128_GCM_SHA256` | | +| `TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA` | | +| `TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA` | | +| `TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA` | | + + +### TLSCipherSuite + + + + + +_Appears in:_ +- [TLSCipherSuites](#tlsciphersuites) + +| Field | Description | +| --- | --- | +| `name` _[TLSCipher](#tlscipher)_ | (Optional) This should be a valid TLS cipher suite name. | + + +### TLSCipherSuites + +_Underlying type:_ _[TLSCipherSuite](#tlsciphersuite)_ + + + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `name` _[TLSCipher](#tlscipher)_ | (Optional) This should be a valid TLS cipher suite name. | + + + + +### TLSPassThroughRouteSpec + + + + + +_Appears in:_ +- [TLSPassThroughRoute](#tlspassthroughroute) + +| Field | Description | +| --- | --- | +| `target` _[TargetType](#targettype)_ | | +| `sniMatch` _[SNIMatch](#snimatch)_ | SNIMatch is used to match requests based on the server name for the intended destination server. Matching requests will be proxied to the Destination. | +| `destination` _string_ | Destination is the destination url to proxy the request to. | + + + + +### TLSTerminatedRouteSpec + + + + + +_Appears in:_ +- [TLSTerminatedRoute](#tlsterminatedroute) + +| Field | Description | +| --- | --- | +| `target` _[TargetType](#targettype)_ | | +| `pathMatch` _[PathMatch](#pathmatch)_ | PathMatch is used to match requests based on what's in the path. Matching requests will be proxied to the Destination defined in this structure. | +| `destination` _string_ | Destination is the destination URL where matching traffic is routed to. | +| `caBundle` _[ConfigMapKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#configmapkeyselector-v1-core)_ | CABundle is where we read the CA bundle from to authenticate the destination (if non-empty) | +| `mtlsCert` _[SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#secretkeyselector-v1-core)_ | (Optional) ForwardingMTLSCert is the certificate used for mTLS between voltron and the destination. Either both ForwardingMTLSCert and ForwardingMTLSKey must be specified, or neither can be specified. | +| `mtlsKey` _[SecretKeySelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#secretkeyselector-v1-core)_ | (Optional) ForwardingMTLSKey is the key used for mTLS between voltron and the destination. Either both ForwardingMTLSCert and ForwardingMTLSKey must be specified, or neither can be specified. | +| `unauthenticated` _boolean_ | (Optional) Unauthenticated says whether the request should go through authentication. This is only applicable if the Target is UI. | + + +### TargetType + +_Underlying type:_ _string_ + + + +_Appears in:_ +- [TLSPassThroughRouteSpec](#tlspassthroughroutespec) +- [TLSTerminatedRouteSpec](#tlsterminatedroutespec) + +| Value | Description | +| --- | --- | +| `UpstreamTunnel` | | +| `UI` | | + + +### TigeraStatus + + + +TigeraStatus represents the most recently observed status for Calico or a Calico Enterprise functional area. + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `TigeraStatus` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[TigeraStatusSpec](#tigerastatusspec)_ | | +| `status` _[TigeraStatusStatus](#tigerastatusstatus)_ | | + + +### TigeraStatusCondition + + + +TigeraStatusCondition represents a condition attached to a particular component. + +_Appears in:_ +- [TigeraStatusStatus](#tigerastatusstatus) + +| Field | Description | +| --- | --- | +| `type` _[StatusConditionType](#statusconditiontype)_ | The type of condition. May be Available, Progressing, or Degraded. | +| `status` _[ConditionStatus](#conditionstatus)_ | The status of the condition. May be True, False, or Unknown. | +| `lastTransitionTime` _[Time](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#time-v1-meta)_ | The timestamp representing the start time for the current status. | +| `reason` _string_ | A brief reason explaining the condition. | +| `message` _string_ | Optionally, a detailed message providing additional context. | +| `observedGeneration` _integer_ | (Optional) observedGeneration represents the generation that the condition was set based upon. For instance, if generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. | + + + + +### TigeraStatusSpec + + + +TigeraStatusSpec defines the desired state of TigeraStatus + +_Appears in:_ +- [TigeraStatus](#tigerastatus) + + + +### TigeraStatusStatus + + + +TigeraStatusStatus defines the observed state of TigeraStatus + +_Appears in:_ +- [TigeraStatus](#tigerastatus) + +| Field | Description | +| --- | --- | +| `conditions` _[TigeraStatusCondition](#tigerastatuscondition) array_ | Conditions represents the latest observed set of conditions for this component. A component may be one or more of Available, Progressing, or Degraded. | + + +### TyphaAffinity + + + +Deprecated. Please use TyphaDeployment instead. +TyphaAffinity allows configuration of node affinity characteristics for Typha pods. + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `nodeAffinity` _[NodeAffinity](#nodeaffinity)_ | (Optional) NodeAffinity describes node affinity scheduling rules for typha. | + + +### TyphaDeployment + + + +TyphaDeployment is the configuration for the typha Deployment. + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[TyphaDeploymentSpec](#typhadeploymentspec)_ | (Optional) Spec is the specification of the typha Deployment. | + + +### TyphaDeploymentContainer + + + +TyphaDeploymentContainer is a typha Deployment container. + +_Appears in:_ +- [TyphaDeploymentPodSpec](#typhadeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the typha Deployment container by name.
Supported values are: calico-typha | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named typha Deployment container's resources. If omitted, the typha Deployment will use its default value for this container's resources. If used in conjunction with the deprecated ComponentResources, then this value takes precedence. | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### TyphaDeploymentInitContainer + + + +TyphaDeploymentInitContainer is a typha Deployment init container. + +_Appears in:_ +- [TyphaDeploymentPodSpec](#typhadeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | Name is an enum which identifies the typha Deployment init container by name.
Supported values are: typha-certs-key-cert-provisioner | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources allows customization of limits and requests for compute resources such as cpu and memory. If specified, this overrides the named typha Deployment init container's resources. If omitted, the typha Deployment will use its default value for this init container's resources. If used in conjunction with the deprecated ComponentResources, then this value takes precedence. | + + +### TyphaDeploymentPodSpec + + + +TyphaDeploymentPodSpec is the typha Deployment's PodSpec. + +_Appears in:_ +- [TyphaDeploymentPodTemplateSpec](#typhadeploymentpodtemplatespec) + +| Field | Description | +| --- | --- | +| `initContainers` _[TyphaDeploymentInitContainer](#typhadeploymentinitcontainer) array_ | (Optional) InitContainers is a list of typha init containers. If specified, this overrides the specified typha Deployment init containers. If omitted, the typha Deployment will use its default values for its init containers. | +| `containers` _[TyphaDeploymentContainer](#typhadeploymentcontainer) array_ | (Optional) Containers is a list of typha containers. If specified, this overrides the specified typha Deployment containers. If omitted, the typha Deployment will use its default values for its containers. | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) Affinity is a group of affinity scheduling rules for the typha pods. If specified, this overrides any affinity that may be set on the typha Deployment. If omitted, the typha Deployment will use its default value for affinity. If used in conjunction with the deprecated TyphaAffinity, then this value takes precedence.
WARNING: Please note that this field will override the default calico-typha Deployment affinity. | +| `nodeSelector` _object (keys:string, values:string)_ | NodeSelector is the calico-typha pod's scheduling constraints. If specified, each of the key/value pairs are added to the calico-typha Deployment nodeSelector provided the key does not already exist in the object's nodeSelector. If omitted, the calico-typha Deployment will use its default value for nodeSelector.
WARNING: Please note that this field will modify the default calico-typha Deployment nodeSelector. | +| `terminationGracePeriodSeconds` _integer_ | (Optional) Optional duration in seconds the pod needs to terminate gracefully. May be decreased in delete request. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). If this value is nil, the default grace period will be used instead. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. Defaults to 30 seconds. | +| `topologySpreadConstraints` _[TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#topologyspreadconstraint-v1-core) array_ | (Optional) TopologySpreadConstraints describes how a group of pods ought to spread across topology domains. Scheduler will schedule pods in a way which abides by the constraints. All topologySpreadConstraints are ANDed. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) Tolerations is the typha pod's tolerations. If specified, this overrides any tolerations that may be set on the typha Deployment. If omitted, the typha Deployment will use its default value for tolerations.
WARNING: Please note that this field will override the default calico-typha Deployment tolerations. | + + +### TyphaDeploymentPodTemplateSpec + + + +TyphaDeploymentPodTemplateSpec is the typha Deployment's PodTemplateSpec + +_Appears in:_ +- [TyphaDeploymentSpec](#typhadeploymentspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[TyphaDeploymentPodSpec](#typhadeploymentpodspec)_ | (Optional) Spec is the typha Deployment's PodSpec. | + + +### TyphaDeploymentSpec + + + +TyphaDeploymentSpec defines configuration for the typha Deployment. + +_Appears in:_ +- [TyphaDeployment](#typhadeployment) + +| Field | Description | +| --- | --- | +| `minReadySeconds` _integer_ | (Optional) MinReadySeconds is the minimum number of seconds for which a newly created Deployment pod should be ready without any of its container crashing, for it to be considered available. If specified, this overrides any minReadySeconds value that may be set on the typha Deployment. If omitted, the typha Deployment will use its default value for minReadySeconds. | +| `template` _[TyphaDeploymentPodTemplateSpec](#typhadeploymentpodtemplatespec)_ | (Optional) Template describes the typha Deployment pod that will be created. | +| `strategy` _[TyphaDeploymentStrategy](#typhadeploymentstrategy)_ | (Optional) The deployment strategy to use to replace existing pods with new ones. | + + +### TyphaDeploymentStrategy + + + +TyphaDeploymentStrategy describes how to replace existing pods with new ones. Only RollingUpdate is supported +at this time so the Type field is not exposed. + +_Appears in:_ +- [TyphaDeploymentSpec](#typhadeploymentspec) + +| Field | Description | +| --- | --- | +| `rollingUpdate` _[RollingUpdateDeployment](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#rollingupdatedeployment-v1-apps)_ | (Optional) Rolling update config params. Present only if DeploymentStrategyType = RollingUpdate. to be. | + + +### WAFExtensionSpec + + + +WAFExtensionSpec configures the WAF Gateway API add-on. + +_Appears in:_ +- [GatewayAPIExtensions](#gatewayapiextensions) + +| Field | Description | +| --- | --- | +| `state` _[WAFExtensionState](#wafextensionstate)_ | (Optional) State turns the WAF Gateway API add-on on or off. Default (nil or "Disabled") means the operator does not render the WAF surface on calico-kube-controllers. Set to "Enabled" to opt in. | + + +### WAFExtensionState + +_Underlying type:_ _string_ + +WAFExtensionState is the on/off enum for the WAF Gateway API add-on. + +_Validation:_ +- Enum: [Enabled Disabled] + + +_Appears in:_ +- [WAFExtensionSpec](#wafextensionspec) + +| Value | Description | +| --- | --- | +| `Enabled` | | +| `Disabled` | | + + +### WAFStatusType + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [Enabled Disabled] + + +_Appears in:_ +- [ApplicationLayerSpec](#applicationlayerspec) + +| Value | Description | +| --- | --- | +| `Disabled` | | +| `Enabled` | | + + +### Whisker + + + + + +| Field | Description | +| --- | --- | +| `apiVersion` _string_ | `operator.tigera.io/v1` | +| `kind` _string_ | `Whisker` | +| `metadata` _[ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#objectmeta-v1-meta)_ | Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[WhiskerSpec](#whiskerspec)_ | | +| `status` _[WhiskerStatus](#whiskerstatus)_ | | + + +### WhiskerDeployment + + + +WhiskerDeployment is the configuration for the whisker Deployment. + +_Appears in:_ +- [WhiskerSpec](#whiskerspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[WhiskerDeploymentSpec](#whiskerdeploymentspec)_ | (Optional) Spec is the specification of the whisker Deployment. | + + +### WhiskerDeploymentContainer + + + + + +_Appears in:_ +- [WhiskerDeploymentPodSpec](#whiskerdeploymentpodspec) + +| Field | Description | +| --- | --- | +| `name` _string_ | | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) | +| `readinessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) ReadinessProbe allows customization of the readiness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | +| `livenessProbe` _[ProbeOverride](#probeoverride)_ | (Optional) LivenessProbe allows customization of the liveness probe timing parameters. The probe handler is set by the operator and cannot be overridden. | + + +### WhiskerDeploymentPodSpec + + + +WhiskerDeploymentPodSpec is the whisker Deployment's PodSpec. + +_Appears in:_ +- [WhiskerDeploymentPodTemplateSpec](#whiskerdeploymentpodtemplatespec) + +| Field | Description | +| --- | --- | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) Affinity is a group of affinity scheduling rules for the whisker pods. | +| `containers` _[WhiskerDeploymentContainer](#whiskerdeploymentcontainer) array_ | (Optional) Containers is a list of whisker containers. If specified, this overrides the specified EGW Deployment containers. If omitted, the whisker Deployment will use its default values for its containers. | +| `nodeSelector` _object (keys:string, values:string)_ | (Optional) NodeSelector gives more control over the nodes where the whisker pods will run on. | +| `terminationGracePeriodSeconds` _integer_ | (Optional) TerminationGracePeriodSeconds defines the termination grace period of the whisker pods in seconds. | +| `topologySpreadConstraints` _[TopologySpreadConstraint](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#topologyspreadconstraint-v1-core) array_ | (Optional) TopologySpreadConstraints describes how a group of pods ought to spread across topology domains. Scheduler will schedule pods in a way which abides by the constraints. All topologySpreadConstraints are ANDed. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) Tolerations is the whisker pod's tolerations. If specified, this overrides any tolerations that may be set on the whisker Deployment. If omitted, the whisker Deployment will use its default value for tolerations. | +| `priorityClassName` _string_ | (Optional) PriorityClassName allows to specify a PriorityClass resource to be used. | + + +### WhiskerDeploymentPodTemplateSpec + + + +WhiskerDeploymentPodTemplateSpec is the whisker Deployment's PodTemplateSpec + +_Appears in:_ +- [WhiskerDeploymentSpec](#whiskerdeploymentspec) + +| Field | Description | +| --- | --- | +| `metadata` _[Metadata](#metadata)_ | (Optional) Refer to Kubernetes API documentation for fields of `metadata`. | +| `spec` _[WhiskerDeploymentPodSpec](#whiskerdeploymentpodspec)_ | (Optional) Spec is the whisker Deployment's PodSpec. | + + +### WhiskerDeploymentSpec + + + +WhiskerDeploymentSpec defines configuration for the whisker Deployment. + +_Appears in:_ +- [WhiskerDeployment](#whiskerdeployment) + +| Field | Description | +| --- | --- | +| `minReadySeconds` _integer_ | (Optional) MinReadySeconds is the minimum number of seconds for which a newly created Deployment pod should be ready without any of its container crashing, for it to be considered available. If specified, this overrides any minReadySeconds value that may be set on the whisker Deployment. If omitted, the whisker Deployment will use its default value for minReadySeconds. | +| `template` _[WhiskerDeploymentPodTemplateSpec](#whiskerdeploymentpodtemplatespec)_ | (Optional) Template describes the whisker Deployment pod that will be created. | +| `strategy` _[WhiskerDeploymentStrategy](#whiskerdeploymentstrategy)_ | (Optional) The deployment strategy to use to replace existing pods with new ones. | + + +### WhiskerDeploymentStrategy + + + + + +_Appears in:_ +- [WhiskerDeploymentSpec](#whiskerdeploymentspec) + +| Field | Description | +| --- | --- | +| `rollingUpdate` _[RollingUpdateDeployment](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#rollingupdatedeployment-v1-apps)_ | (Optional) Rolling update config params. Present only if DeploymentStrategyType = RollingUpdate. to be. | + + +### WhiskerSpec + + + + + +_Appears in:_ +- [Whisker](#whisker) + +| Field | Description | +| --- | --- | +| `whiskerDeployment` _[WhiskerDeployment](#whiskerdeployment)_ | | +| `notifications` _[NotificationMode](#notificationmode)_ | (Optional)
Default: Enabled This setting enables calls to an external API to retrieve notification banner text in the Whisker UI. Allowed values are Enabled or Disabled. Defaults to Enabled. | + + +### WhiskerStatus + + + +WhiskerStatus defines the observed state of Whisker + +_Appears in:_ +- [Whisker](#whisker) + +| Field | Description | +| --- | --- | +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#condition-v1-meta) array_ | (Optional) Conditions represents the latest observed set of conditions for the component. A component may be one or more of Ready, Progressing, Degraded or other customer types. | + + +### WindowsDataplaneOption + +_Underlying type:_ _string_ + + + +_Validation:_ +- Enum: [HNS Disabled] + + +_Appears in:_ +- [CalicoNetworkSpec](#caliconetworkspec) + +| Value | Description | +| --- | --- | +| `Disabled` | | +| `HNS` | | + + +### WindowsNodeSpec + + + + + +_Appears in:_ +- [InstallationSpec](#installationspec) + +| Field | Description | +| --- | --- | +| `cniBinDir` _string_ | (Optional) CNIBinDir is the path to the CNI binaries directory on Windows, it must match what is used as 'bin_dir' under [plugins] [plugins."io.containerd.grpc.v1.cri"] [plugins."io.containerd.grpc.v1.cri".cni] on the containerd 'config.toml' file on the Windows nodes. | +| `cniConfigDir` _string_ | (Optional) CNIConfigDir is the path to the CNI configuration directory on Windows, it must match what is used as 'conf_dir' under [plugins] [plugins."io.containerd.grpc.v1.cri"] [plugins."io.containerd.grpc.v1.cri".cni] on the containerd 'config.toml' file on the Windows nodes. | +| `cniLogDir` _string_ | (Optional) CNILogDir is the path to the Calico CNI logs directory on Windows. | +| `vxlanMACPrefix` _string_ | (Optional) VXLANMACPrefix is the prefix used when generating MAC addresses for virtual NICs | +| `vxlanAdapter` _string_ | (Optional) VXLANAdapter is the Network Adapter used for VXLAN, leave blank for primary NIC | + + +### ZTunnelDaemonset + + + +ZTunnelDaemonset defines customized settings for the ZTunnelDaemonset component. + +_Appears in:_ +- [IstioSpec](#istiospec) + +| Field | Description | +| --- | --- | +| `spec` _[ZTunnelDaemonsetSpec](#ztunneldaemonsetspec)_ | (Optional) Spec allows users to specify custom fields for the ZTunnel Daemonset. | + + +### ZTunnelDaemonsetPodSpec + + + +ZTunnelDaemonsetPodSpec defines the pod spec for customizing the ZTunnel Daemonset. + +_Appears in:_ +- [ZTunnelDaemonsetSpecTemplate](#ztunneldaemonsetspectemplate) + +| Field | Description | +| --- | --- | +| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#affinity-v1-core)_ | (Optional) Affinity specifies the affinity for the deployment. | +| `nodeSelector` _object (keys:string, values:string)_ | (Optional) NodeSelector specifies the node affinity for the deployment. | +| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#resourcerequirements-v1-core)_ | (Optional) Resources specifies the compute resources required for the deployment. | +| `tolerations` _[Toleration](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.32/#toleration-v1-core) array_ | (Optional) Tolerations specifies the tolerations for the deployment. | + + +### ZTunnelDaemonsetSpec + + + +ZTunnelDaemonsetSpec defines the spec for customizing the ZTunnel Daemonset. + +_Appears in:_ +- [ZTunnelDaemonset](#ztunneldaemonset) + +| Field | Description | +| --- | --- | +| `template` _[ZTunnelDaemonsetSpecTemplate](#ztunneldaemonsetspectemplate)_ | (Optional) Template allows users to specify custom fields for the ZTunnel Daemonset. | + + +### ZTunnelDaemonsetSpecTemplate + + + +ZTunnelDaemonsetSpecTemplate defines the template for customizing the ZTunnel Daemonset. + +_Appears in:_ +- [ZTunnelDaemonsetSpec](#ztunneldaemonsetspec) + +| Field | Description | +| --- | --- | +| `spec` _[ZTunnelDaemonsetPodSpec](#ztunneldaemonsetpodspec)_ | (Optional) Spec allows users to specify custom fields for the ZTunnel Daemonset. | + diff --git a/calico-cloud_versioned_docs/version-24-1/reference/installation/_crd-ref-docs/config.yaml b/calico-cloud_versioned_docs/version-24-1/reference/installation/_crd-ref-docs/config.yaml new file mode 100644 index 0000000000..0ac6bb36c2 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/installation/_crd-ref-docs/config.yaml @@ -0,0 +1,86 @@ +processor: + # RE2 regular expressions describing types that should be excluded from the generated documentation. + ignoreTypes: + - "List$" + - "Tenant*" + - "Compliance" + - "ManagementCluster$" + - "ManagementClusterSpec$" + - '\.TLS$' + - "Manager" + - "ManagerSpec" + - "ManagerStatus" + - "ManagerDeployment" + - "ManagerDeploymentSpec" + - "ManagerDeploymentPodTemplateSpec" + - "ManagerDeploymentPodSpec" + - "ManagerDeploymentInitContainer" + - "ManagerDeploymentContainer" + - "Authentication" + - "AuthenticationSpec" + - "AuthenticationStatus" + - "AuthenticationOIDC" + - "EmailVerificationType" + - "PromptType" + - "OIDCType" + - "AuthenticationOpenshift" + - "AuthenticationLDAP" + - "UserSearch" + - "GroupSearch" + - "UserMatch" + - "DexDeployment" + - "DexDeploymentSpec" + - "DexDeploymentPodTemplateSpec" + - "DexDeploymentPodSpec" + - "DexDeploymentInitContainer" + - "LogStorage" + - "LogStorageStatus" + - "Nodes" + - "NodeSet" + - "NodeSetSelectionAttribute" + - "Indices" + - "Retention" + - "LogStorageComponentResource" + - "LogStorageComponentName" + - "ECKOperatorStatefulSet" + - "ECKOperatorStatefulSetSpec" + - "ECKOperatorStatefulSetPodTemplateSpec" + - "ECKOperatorStatefulSetPodSpec" + - "ECKOperatorStatefulSetInitContainer" + - "ECKOperatorStatefulSetContainer" + - "LinseedDeployment" + - "LinseedDeploymentSpec" + - "LinseedDeploymentPodTemplateSpec" + - "LinseedDeploymentPodSpec" + - "LinseedDeploymentInitContainer" + - "LinseedDeploymentContainer" + - "ElasticsearchMetricsDeployment" + - "ElasticsearchMetricsDeploymentSpec" + - "ElasticsearchMetricsDeploymentPodTemplateSpec" + - "ElasticsearchMetricsDeploymentPodSpec" + - "ElasticsearchMetricsDeploymentInitContainer" + - "ElasticsearchMetricsDeploymentContainer" + - "ESGatewayDeployment" + - "ESGatewayDeploymentSpec" + - "ESGatewayDeploymentPodTemplateSpec" + - "ESGatewayDeploymentPodSpec" + - "ESGatewayDeploymentInitContainer" + - "ESGatewayDeploymentContainer" + - "Kibana" + - "DashboardsJob" + - "DataType$" + # RE2 regular expressions describing type fields that should be excluded from the generated documentation. + ignoreFields: + - "TypeMeta$" + customMarkers: + - name: "optional" + target: field + +render: + # Version of Kubernetes to use when generating links to Kubernetes API documentation. + kubernetesVersion: 1.32 + # Generate better link for known types + #knownTypes: + # - name: SecretObjectReference + # package: sigs.k8s.io/gateway-api/apis/v1beta1 + # link: https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.SecretObjectReference diff --git a/calico-cloud_versioned_docs/version-24-1/reference/installation/_crd-ref-docs/templates/gv_details.tpl b/calico-cloud_versioned_docs/version-24-1/reference/installation/_crd-ref-docs/templates/gv_details.tpl new file mode 100644 index 0000000000..3bd48c66bc --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/installation/_crd-ref-docs/templates/gv_details.tpl @@ -0,0 +1,21 @@ +{{- define "gvDetails" -}} +{{- $gv := . -}} + +{/* vale off */} + +## {{ $gv.GroupVersionString }} + +{{ $gv.Doc }} + +{{- if $gv.Kinds }} +Resource Types +{{- range $gv.SortedKinds }} +- {{ $gv.TypeForKind . | markdownRenderTypeLink }} +{{- end }} +{{ end }} + +{{ range $gv.SortedTypes }} +{{ template "type" . }} +{{ end }} + +{{- end -}} diff --git a/calico-cloud_versioned_docs/version-24-1/reference/installation/_crd-ref-docs/templates/gv_list.tpl b/calico-cloud_versioned_docs/version-24-1/reference/installation/_crd-ref-docs/templates/gv_list.tpl new file mode 100644 index 0000000000..30ad594057 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/installation/_crd-ref-docs/templates/gv_list.tpl @@ -0,0 +1,13 @@ +{{- define "gvList" -}} +{{- $groupVersions := . -}} + +Packages +{{- range $groupVersions }} +- {{ markdownRenderGVLink . }} +{{- end }} + +{{ range $groupVersions }} +{{ template "gvDetails" . -}} +{{ end }} + +{{- end -}} diff --git a/calico-cloud_versioned_docs/version-24-1/reference/installation/_crd-ref-docs/templates/type.tpl b/calico-cloud_versioned_docs/version-24-1/reference/installation/_crd-ref-docs/templates/type.tpl new file mode 100644 index 0000000000..bcf45a0967 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/installation/_crd-ref-docs/templates/type.tpl @@ -0,0 +1,51 @@ +{{- define "type" -}} +{{- $type := . -}} +{{- if markdownShouldRenderType $type -}} + +### {{ $type.Name }} + +{{ if $type.IsAlias }}_Underlying type:_ _{{ markdownRenderTypeLink $type.UnderlyingType }}_{{ end }} + +{{ $type.Doc }} + +{{- if $type.Validation }} + +_Validation:_ +{{- range $type.Validation }} +- {{ . }} +{{ end }} +{{- end -}} + +{{- if $type.References }} + +_Appears in:_ +{{- range $type.SortedReferences }} +- {{ markdownRenderTypeLink . }} +{{- end }} +{{- end }} + +{{ if $type.Members -}} +| Field | Description | +| --- | --- | +{{ if $type.GVK -}} +| `apiVersion` _string_ | `{{ $type.GVK.Group }}/{{ $type.GVK.Version }}` | +| `kind` _string_ | `{{ $type.GVK.Kind }}` | +{{ end -}} + +{{ range $type.Members -}} +| `{{ .Name }}` _{{ markdownRenderType .Type }}_ | {{ with .Markers.optional -}}(Optional) {{ end -}} {{ template "type_members" . }} | +{{ end -}} + +{{ end -}} + +{{ if $type.EnumValues -}} +| Value | Description | +| --- | --- | +{{ range $type.EnumValues -}} +| `{{ .Name }}` | {{ markdownRenderFieldDoc .Doc }} | +{{ end -}} +{{ end -}} + + +{{- end -}} +{{- end -}} diff --git a/calico-cloud_versioned_docs/version-24-1/reference/installation/_crd-ref-docs/templates/type_members.tpl b/calico-cloud_versioned_docs/version-24-1/reference/installation/_crd-ref-docs/templates/type_members.tpl new file mode 100644 index 0000000000..041758a872 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/installation/_crd-ref-docs/templates/type_members.tpl @@ -0,0 +1,8 @@ +{{- define "type_members" -}} +{{- $field := . -}} +{{- if eq $field.Name "metadata" -}} +Refer to Kubernetes API documentation for fields of `metadata`. +{{- else -}} +{{ markdownRenderFieldDoc $field.Doc }} +{{- end -}} +{{- end -}} diff --git a/calico-cloud_versioned_docs/version-24-1/reference/installation/api.mdx b/calico-cloud_versioned_docs/version-24-1/reference/installation/api.mdx new file mode 100644 index 0000000000..69c4378ce3 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/installation/api.mdx @@ -0,0 +1,17 @@ +--- +description: Tigera Operator API reference for Calico Cloud, covering all operator.tigera.io custom resources for installing and managing your connected cluster. +--- + +# Tigera Operator API reference + +import API from '@site/calico-cloud_versioned_docs/version-24-1/reference/installation/_api.mdx'; + +The Tigera Operator installs and manages $[prodname] by watching the custom resources in the `operator.tigera.io/v1` API group. +This page documents the resources in that group that you can configure in $[prodname]. +The `Installation` resource sets cluster-wide options at install time, and other resources, such as `APIServer` and `IntrusionDetection`, configure individual components. +You can change most settings on a running cluster by using `kubectl`, and the operator reconciles the cluster to match. + +This API group is separate from the `projectcalico.org/v3` resources that you use for day-to-day networking and policy. +For those, see [Resource definitions](../resources/overview.mdx). + + diff --git a/calico-cloud_versioned_docs/version-24-1/reference/installation/config.json b/calico-cloud_versioned_docs/version-24-1/reference/installation/config.json new file mode 100644 index 0000000000..3f061af91d --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/installation/config.json @@ -0,0 +1,36 @@ +{ + "hideMemberFields": [ + "TypeMeta" + ], + "hideTypePatterns": [ + "ParseError$", + "List$", + "ImageAssuranceCentral$", + "ImageAssuranceCentralSpec$", + "ImageAssuranceCentralStatus$", + "APIProxyDeployment", + "ManagedClusterControllerDeployment", + "RuntimeCleanerDeployment", + "ScannerWorkerDeployment", + "WaiterDeployment" + ], + "externalPackages": [ + { + "typeMatchPrefix": "^k8s\\.io/apimachinery/pkg/apis/meta/v1\\.Duration$", + "docsURLTemplate": "https://godoc.org/k8s.io/apimachinery/pkg/apis/meta/v1#Duration" + }, + { + "typeMatchPrefix": "^k8s\\.io/(api|apimachinery/pkg/apis)/", + "docsURLTemplate": "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.31/#{{lower .TypeIdentifier}}-{{arrIndex .PackageSegments -1}}-{{arrIndex .PackageSegments -2}}" + }, + { + "typeMatchPrefix": "^github\\.com/knative/pkg/apis/duck/", + "docsURLTemplate": "https://godoc.org/github.com/knative/pkg/apis/duck/{{arrIndex .PackageSegments -1}}#{{.TypeIdentifier}}" + } + ], + "typeDisplayNamePrefixOverrides": { + "k8s.io/api/": "Kubernetes ", + "k8s.io/apimachinery/pkg/apis/": "Kubernetes " + }, + "markdownDisabled": false +} diff --git a/calico-cloud_versioned_docs/version-24-1/reference/public-cloud/aws.mdx b/calico-cloud_versioned_docs/version-24-1/reference/public-cloud/aws.mdx new file mode 100644 index 0000000000..87f1dd95a5 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/public-cloud/aws.mdx @@ -0,0 +1,176 @@ +--- +description: Reference for connecting Calico Cloud clusters running on Amazon Web Services covering supported networking modes and AWS platform notes. +--- + +# Amazon Web Services + +$[prodname] provides the following advantages when running in Amazon Web Services (AWS): + +- **Network Policy for Containers**: $[prodname] provides fine-grained network security policy for individual containers. +- **No Overlays**: Within each VPC subnet $[prodname] doesn't need an overlay, which means high performance networking for your containers. +- **No 50 Node Limit**: $[prodname] allows you to surpass the 50 node limit, which exists as a consequence of the [AWS 50 route limit](http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_Appendix_Limits.html#vpc-limits-route-tables) when using the VPC routing table. + +## Routing traffic within a single VPC subnet + +Since $[prodname] assigns IP addresses outside the range used by AWS for EC2 instances, you must disable AWS src/dst +checks on each EC2 instance in your cluster +[as described in the AWS documentation](http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/VPC_NAT_Instance.html#EIP_Disable_SrcDestCheck). This +allows $[prodname] to route traffic natively within a single VPC subnet without using an overlay or any of the limited VPC routing table entries. + +## Routing traffic across different VPC subnets / VPCs + +If you need to split your deployment across multiple AZs for high availability then each AZ will have its own VPC subnet. To +use $[prodname] across multiple different VPC subnets or [peered VPCs](http://docs.aws.amazon.com/AmazonVPC/latest/UserGuide/vpc-peering.html), +in addition to disabling src/dst checks as described above you must also enable IPIP encapsulation and outgoing NAT +on your $[prodname] IP pools. + +See the [IP pool configuration reference](../resources/ippool.mdx) +for information on how to configure $[prodname] IP pools. + +By default, $[prodname]'s IPIP encapsulation applies to all container-to-container traffic. However, +encapsulation is only required for container traffic that crosses a VPC subnet boundary. For better +performance, you can configure $[prodname] to perform IPIP encapsulation only across VPC subnet boundaries. + +To enable the "CrossSubnet" IPIP feature, configure your $[prodname] IP pool resources +to enable IPIP and set the mode to "CrossSubnet". + +:::note + +This feature was introduced in $[prodname] v2.1, if your deployment was created with +an older version of $[prodname], or if you if you are unsure whether your deployment +is configured correctly, follow the [Configuring IP-in-IP guide](../../networking/configuring/vxlan-ipip.mdx) +which discusses this in more detail. + +::: + +The following `kubectl` command will create or modify an IPv4 pool with +CIDR 192.168.0.0/16 using IPIP mode `CrossSubnet`. Adjust the pool CIDR for your deployment. + +```bash +kubectl apply -f - < diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/alertexception.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/alertexception.mdx new file mode 100644 index 0000000000..e34d0f1cb2 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/alertexception.mdx @@ -0,0 +1,78 @@ +--- +description: Reference for the AlertException resource in Calico Cloud connected clusters that suppresses alerts matching a defined filter for a time window. +--- + +# Alert exception + +An alert exception resource is a filter that hides specific alerts from users in the $[prodname] web console. +You can filter alerts by time range or indefinitely. If an alert exception expires, alerts will reappear in the web console. + +For `kubectl` [commands](https://kubernetes.io/docs/reference/kubectl/overview/), +the following case-insensitive aliases can be used to specify the resource type on the CLI: +`alertexception.projectcalico.org`, `alertexceptions.projectcalico.org` and abbreviations such as +`alertexception.p` and `alertexceptions.p`. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: AlertException +metadata: + name: sample +spec: + description: 'Sample alert exception' + selector: origin = "" and source_namespace = "" + startTime: '2022-01-02T00:00:00Z' + endTime: '2022-01-03T00:00:00Z' +``` + +## Alert exception definition + +| Field | Description | Accepted Values | Schema | +| ----- | --------------------------------- | --------------------------------------------------- | ------ | +| name | The name of this alert exception. | Alphanumeric string with optional `.`, `_`, or `-`. | string | + +### Spec + +| Field | Description | Type | Required | Acceptable Values | +| ----------- | ----------------------------------------------------------------------------------- | ----------------------- | -------- | ----------------------- | +| description | Human-readable description of the alert exception. | string | yes | +| selector | Selects alerts to filter from the $[prodname] web console queries. | string | yes | [selector](#selector) | +| startTime | Defines the start time from which this alert exception will start filtering alerts. | Date in RFC 3339 format | yes | [startTime](#starttime) | +| endTime | Defines the end time at which this alert exception will stop filtering alerts. | Date in RFC 3339 format | | [endTime](#endtime) | + +### Selector + +A selector is an expression that matches alerts based on their fields. For each alert, +`origin` and `type` fields are automatically set by the applicable component, but other fields can be empty. + +| Field | Description | +| ---------------- | ---------------------------------------------------------------------------------- | +| origin | User specified or generated names from $[prodname] threat defense components. | +| type | $[prodname] threat defense components an alert is generated from. | +| host | Name of the node that triggers this alert. | +| dest_ip | IP address of the destination pod. | +| dest_name | Name of the destination pod. | +| dest_name_aggr | Aggregated name of the destination pod. | +| dest_namespace | Namespace of the destination endpoint. A `-` means the endpoint is not namespaced. | +| source_ip | IP address of the source pod. | +| source_name | Name of the source pod. | +| source_name_aggr | Aggregated name of the source pod. | +| source_namespace | Namespace of the source endpoint. A `-` means the endpoint is not namespaced. | + +The selector also supports logical operators, which can be combined into larger expressions. + +| Expression | Meaning | +| ----------------------------------- | ----------------------------------------------------------------------------- | +| ` AND ` | Matches if and only if both ``, and, `` matches | +| ` OR ` | Matches if and only if either ``, or, `` matches. | + +### StartTime + +Defines the start time when this alert exception starts filtering alerts in RFC 3339 format. This value is required. + +### EndTime + +Defines the end time when this alert exception stops filtering alerts in RFC 3339 format. +If omitted, alerts are filtered indefinitely. +If the value is changed to the past, this alert exception is disabled immediately. diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/bfdconfig.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/bfdconfig.mdx new file mode 100644 index 0000000000..936d1e6431 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/bfdconfig.mdx @@ -0,0 +1,63 @@ +--- +description: Reference for the BFD configuration resource in Calico Cloud connected clusters that tunes Bidirectional Forwarding Detection on BGP-peered nodes. +--- + +import Selectors from '@site/calico-enterprise/_includes/content/_selectors.mdx'; + +# BFD configuration + +A bidirectional forwarding detection (BFD) configuration resource (`BFDConfiguration`) represents BFD-specific configuration options for a selection of one or more nodes. + +For `kubectl` commands, the following case-insensitive aliases may be used to specify the resource type on the CLI: `bfdconfiguration.projectcalico.org`, `bfdconfigurations.projectcalico.org` as well as abbreviations such as `bfdconfiguration.p` and `bfdconfigurations.p`. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: BFDConfiguration +metadata: + name: default +spec: + nodeSelector: "all()" + interfaces: + - matchPattern: "*" + minimumRecvInterval: 10ms + minimumSendInterval: 100ms + idleSendInterval: 1m + multiplier: 5 +``` + +## BFD configuration definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | --------------------------------------------------------- | --------------------------------------------------- | ------ | +| name | Unique name to describe this resource instance. Required. | Alphanumeric string with optional `.`, `_`, or `-`. | string | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ------- | ------------- | --------------- | ------- | ------- | +| nodeSelector | Selects one or more nodes to which this resource applies. | | string | [selector](#selector) | +| interfaces | List of per-interface BFD configuration parameters. | | List of [BFDInterface](#bfdinterface) | | + +### BFDInterface + +| Field | Description | Accepted Values | Schema | Default | +| ------- | ------------- | --------------- | ------- | ------- | +| matchPattern | Pattern to match one or more interfaces. Supports exact interface names, match on interface prefixes (e.g., “eth*”), or “*” to select all interfaces on the selected node(s). | | string | | +| minimumRecvInterval | The minimum interval between received BFD packets. Must be a whole number of milliseconds greater than 0. | | string | 10ms | +| minimumSendInterval | The minimum interval between transmitted BFD packets. Must be a whole number of milliseconds greater than 0. | | string | 100ms | +| idleSendInterval | The interval between transmitted BFD packets when the BFD peer is idle. Must be a whole number of milliseconds greater than 0. | | string | 1m | +| multiplier | The number of intervals that must pass without receiving a BFD packet before the peer is considered down. | | string | 5 | + +### Selector + + + +## Supported operations + +| Datastore type | Create | Delete | Update | Get/List | Notes | +| --------------------- | ------ | ------ | ------ | -------- | ----- | +| Kubernetes API server | Yes | Yes | Yes | Yes | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/bgpconfig.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/bgpconfig.mdx new file mode 100644 index 0000000000..4913e97d04 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/bgpconfig.mdx @@ -0,0 +1,90 @@ +--- +description: Reference for the BGPConfiguration resource in Calico Cloud connected clusters that sets cluster-wide BGP options including route reflectors and AS number. +--- + +# BGP configuration + +A Border Gateway Protocol (BGP) configuration resource (`BGPConfiguration`) represents BGP configuration options +for the cluster or a specific node. + +For `kubectl` commands, the following case-insensitive aliases may be used to specify the resource type on the CLI: `bgpconfiguration.projectcalico.org`, `bgpconfigurations.projectcalico.org` as well as abbreviations such as `bgpconfiguration.p` and `bgpconfigurations.p`. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: BGPConfiguration +metadata: + name: default +spec: + logSeverityScreen: Info + nodeToNodeMeshEnabled: true + nodeMeshMaxRestartTime: 120s + asNumber: 63400 + serviceClusterIPs: + - cidr: 10.96.0.0/12 + serviceExternalIPs: + - cidr: 104.244.42.129/32 + - cidr: 172.217.3.0/24 + listenPort: 178 + bindMode: NodeIP + communities: + - name: bgp-large-community + value: 63400:300:100 + prefixAdvertisements: + - cidr: 172.218.4.0/26 + communities: + - bgp-large-community + - 63400:120 +``` + +## BGP configuration definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | --------------------------------------------------------- | --------------------------------------------------- | ------ | +| name | Unique name to describe this resource instance. Required. | Alphanumeric string with optional `.`, `_`, or `-`. | string | + +- The resource with the name `default` has a specific meaning - this contains the BGP global default configuration. +- The resources with the name `node.` contain the node-specific overrides, and will be applied to the node ``. When deleting a node the BGPConfiguration resource associated with the node will also be deleted. Only prefixAdvertisements, listenPort, and logSeverityScreen can be overridden this way. + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------- | ----------------------------------------------------- | --------------------------------------------------------------- | +| logSeverityScreen | Global log level | Debug, Info, Warning, Error, Fatal | string | `Info` | +| nodeToNodeMeshEnabled | Full BGP node-to-node mesh. Only valid on the global `default` BGPConfiguration. | true, false | string | true | +| asNumber | The default local AS Number that $[prodname] should use when speaking with BGP peers. Only valid on the global `default` BGPConfiguration; to set a per-node override, use the `bgp` field on the [Node resource](node.mdx). | A valid AS Number, may be specified in dotted notation. | integer/string | 64512 | +| extensions | Additional mapping of keys and values. Used for setting values in custom BGP configurations. | valid strings for both keys and values | map | | +| serviceClusterIPs | The CIDR blocks for Kubernetes Service Cluster IPs to be advertised over BGP. Only valid on the global `default` BGPConfiguration: will be ignored otherwise. | A list of valid IPv4 or IPv6 CIDR blocks. | List of `cidr: /` values. | Empty List | +| serviceExternalIPs | The CIDR blocks for Kubernetes Service External IPs to be advertised over BGP. Kubernetes Service External IPs will only be advertised if they are within one of these blocks. Only valid on the global `default` BGPConfiguration: will be ignored otherwise. | A list of valid IPv4 or IPv6 CIDR blocks. | List of `cidr: /` values. | Empty List | +| serviceLoadBalancerIPs | The CIDR blocks for Kubernetes Service status.LoadBalancer IPs to be advertised over BGP. Kubernetes LoadBalancer IPs will only be advertised if they are within one of these blocks. Only valid on the global `default` BGPConfiguration: will be ignored otherwise. | A list of valid IPv4 or IPv6 CIDR blocks. | List of `cidr: /` values. | Empty List | +| listenPort | The port where BGP protocol should listen. | A valid port number. | integer | 179 | +| bindMode | Indicates whether to listen for BGP connections on all addresses (None) or only on the node's canonical IP address Node.Spec.BGP.IPvXAddress (NodeIP). If this field is changed when calico-node is already running, the change will not take effect until calico-node is manually restarted. | None, NodeIP. | string | None | +| communities | List of BGP community names and their values, communities are not advertised unless they are used in [prefixAdvertisements](#prefixadvertisements). | | List of [communities](#communities) | +| prefixAdvertisements | List of per-prefix advertisement properties, like BGP communities. | | List of [prefixAdvertisements](#prefixadvertisements) | +| nodeMeshPassword | BGP password for the all the peerings in a full mesh configuration. | | [BGPPassword](bgppeer.mdx#bgppassword) | `nil` (no password) | +| nodeMeshMaxRestartTime | Restart time that is announced by BIRD in the BGP graceful restart capability and that specifies how long the neighbor would wait for the BGP session to re-establish after a restart before deleting stale routes in full mesh configurations. Note: extra care should be taken when changing this configuration, as it may break networking in your cluster. When not specified, BIRD uses the default value of 120 seconds. | `10s`, `120s`, `2m` etc. | [Duration string][parse-duration] | `nil` (empty config, BIRD will use the default value of `120s`) | +| localWorkloadPeeringIPV4 | Single virtual IPv4 address that all nodes will use for peering with local workloads when using `BGPPeer`'s `localWorkloadSelector`. A link local address is recommended but it must not be 169.254.0.1 or 169.254.0.2 because these are used internally by $[prodname]. See the [guide](../../networking/configuring/bgp-to-workload.mdx) for details. | An IPv4 address. | string | None | +| localWorkloadPeeringIPV6 | Single virtual IPv6 address that all nodes will use for peering with local workloads when using `BGPPeer`'s `localWorkloadSelector`. Must _not_ be a link-local address (because $[prodname] doesn't currently support the required "scope" suffix). See the [guide](../../networking/configuring/bgp-to-workload.mdx) for details. | An IPv6 address. | string | None | + +### communities + +| Field | Description | Accepted Values | Schema | +| ----- | -------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | +| name | Name or identifier for the community. This should be used in [prefixAdvertisements](#prefixadvertisements) to advertise the community value. | | string | +| value | Standard or large BGP community value. | For standard community, value should be in `aa:nn` format, where both `aa` and `nn` are 16 bit integers.
For large community, value should be `aa:nn:mm` format, where `aa`, `nn` and `mm` are all 32 bit integers.
Where `aa` is an AS Number, `nn` and `mm` are per-AS identifier. | string | + +### prefixAdvertisements + +| Field | Description | Accepted Values | Schema | +| ----------- | ----------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------- | +| cidr | CIDR for which properties should be advertised. | `cidr: XXX.XXX.XXX.XXX/XX` | string | +| communities | BGP communities to be advertised. | Communities can be list of either community names already defined in [communities](#communities) or community value of format `aa:nn` or `aa:nn:mm`.
For standard community, value should be in `aa:nn` format, where both `aa` and `nn` are 16 bit integers.
For large community, value should be `aa:nn:mm` format, where `aa`, `nn` and `mm` are all 32 bit integers.
Where `aa` is an AS Number, `nn` and `mm` are per-AS identifier. | List of string | + +## Supported operations + +| Datastore type | Create | Delete | Delete (Global `default`) | Update | Get/List | Notes | +| --------------------- | ------ | ------ | ------------------------- | ------ | -------- | ----- | +| Kubernetes API server | Yes | Yes | No | Yes | Yes | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/bgpfilter.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/bgpfilter.mdx new file mode 100644 index 0000000000..04d556a89d --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/bgpfilter.mdx @@ -0,0 +1,102 @@ +--- +description: Reference for the BGPFilter resource in Calico Cloud connected clusters that filters routes imported from or exported to BGP peers. +--- + +# BGP filter + +A Border Gateway Protocol (BGP) filter resource (`BGPFilter`) represents a way to control +routes imported by and exported to BGP peers +specified by a [BGP peer resource](bgppeer.mdx) (`BGPPeer`). + +The BGPFilter rules are applied sequentially: the `action` for +the **first** rule that matches is executed immediately. +If an address does not match any explicit BGP filter rule, +the default action is `Accept`. + +In order for a BGPFilter to be used in a BGP peering, its `name` +must be added to `filters` of the corresponding BGPPeer resource. + +For `kubectl` commands, the following case-sensitive aliases may +be used to specify the resource type on the CLI: `bgpfilters.projectcalico.org` + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: BGPFilter +metadata: + name: my-filter +spec: + exportV4: + - action: Accept + matchOperator: In + cidr: 77.0.0.0/16 + source: RemotePeers + - action: Reject + interface: '*.calico' + importV4: + - action: Accept + matchOperator: In + cidr: 55.0.0.0/16 + prefixLength: + min: 30 + - action: Reject + matchOperator: NotIn + cidr: 44.0.0.0/16 + exportV6: + - action: Reject + source: RemotePeers + - action: Reject + interface: '*.calico' + importV6: + - action: Accept + matchOperator: Equal + cidr: 5000::0/64 + - action: Reject +``` + +## BGP filter definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | ------------------------------------------------------------------ | --------------------------------------------------- | ------ | +| name | Unique name to describe this resource instance. Must be specified. | Alphanumeric string with optional `.`, `_`, or `-`. | string | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| -------- | ---------------------------------- | --------------- | ----------------------------------------- | ------- | +| exportV4 | List of v4 CIDRs and export action | | [BGP Filter Rule v4](#bgp-filter-rule-v4) | | +| importV4 | List of v4 CIDRs and import action | | [BGP Filter Rule v4](#bgp-filter-rule-v4) | | +| exportV6 | List of v6 CIDRs and export action | | [BGP Filter Rule v6](#bgp-filter-rule-v6) | | +| importV6 | List of v6 CIDRs and import action | | [BGP Filter Rule v6](#bgp-filter-rule-v6) | | + +### BGP Filter Rule v4 + +| Field | Description | Accepted Values | Schema | Default | +| ------------- | ------------------------------------------ | ------------------------------------------------------------------- | ------ | ------- | +| cidr | IPv6 range | A valid IPv6 CIDR | string | | +| prefixLength | PrefixLength | Valid integers between 0 and ipv4/6 max (32, 128) | | | +| matchOperator | Method by which to match candidate routes | `In`, `NotIn`, `Equal`, `NotEqual` | string | | +| source | Indicator of the source of route | `RemotePeers` means any route learned from other BGP peers | string | | +| interface | String to match interface names | A valid pattern to match interfaces. "*" can be used as a wildcard. | string | | +| action | Action to be taken for this rule | `Accept` or `Reject` | string | | + +### BGP Filter Rule v6 + +| Field | Description | Accepted Values | Schema | Default | +| ------------- | ------------------------------------------ | ------------------------------------------------------------------- | ------ | ------- | +| cidr | IPv6 range | A valid IPv6 CIDR | string | | +| prefixLength | PrefixLength | Valid integers between 0 and ipv4/6 max (32, 128) | | | +| matchOperator | Method by which to match candidate routes | `In`, `NotIn`, `Equal`, `NotEqual` | string | | +| source | Indicator of the source of route | `RemotePeers` means any route learned from other BGP peers | string | | +| interface | String to match interface names | A valid pattern to match interfaces. "*" can be used as a wildcard. | string | | +| action | Action to be taken for this rule | `Accept` or `Reject` | string | | + + +## Supported operations + +| Datastore type | Create/Delete | Update | Get/List | Notes | +| --------------------- | ------------- | ------ | -------- | ----- | +| Kubernetes API server | Yes | Yes | Yes | | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/bgppeer.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/bgppeer.mdx new file mode 100644 index 0000000000..68b5e8a257 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/bgppeer.mdx @@ -0,0 +1,195 @@ +--- +description: Reference for the BGPPeer resource in Calico Cloud connected clusters that defines BGP neighbor relationships with external routers or other Calico nodes. +--- + +# BGP peer + +import Selectors from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_selectors.mdx'; + +A Border Gateway Protocol (BGP) peer resource (`BGPPeer`) represents a remote BGP peer +with which one or more nodes in a $[prodname] cluster exchange routes. +Configuring BGP peers allows you to peer a $[prodname] network +with your data center fabric, such as a top-of-rack (ToR) router. For more +information on cluster layouts, see $[prodname]'s documentation on +[$[prodname] over IP fabrics](../architecture/design/l3-interconnect-fabric.mdx). + +For `kubectl` commands, the following case-insensitive aliases may be used to specify the resource type on the CLI: `bgppeer.projectcalico.org`, `bgppeers.projectcalico.org` as well as abbreviations such as `bgppeer.p` and `bgppeers.p`. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: BGPPeer +metadata: + name: some.name +spec: + node: rack1-host1 + peerIP: 192.168.1.1 + asNumber: 63400 +``` + +## BGP peer definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | ------------------------------------------------------------------ | --------------------------------------------------- | ------ | +| name | Unique name to describe this resource instance. Must be specified. | Alphanumeric string with optional `.`, `_`, or `-`. | string | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | --------------------------- | ---------------------------------------------------------------------------- | +| node | If specified, the scope is node level, otherwise the scope is global. | The hostname of the node to which this peer applies. | string | | +| peerIP | The IP address of this peer and an optional port number. If port number is not set, and peer is Calico node with `listenPort` set, then `listenPort` is used. | Valid IPv4 or IPv6 address. If port number is set use, `IPv4:port` or `[IPv6]:port` format. | string | | +| asNumber | The remote AS Number of the peer. | A valid AS Number, may be specified in dotted notation. | integer/string | +| nodeSelector | Selector for the nodes that should have this peering. When this is set, the `node` field must be empty. | | [selector](#selector) | +| peerSelector | Selector for the remote nodes to peer with. When this is set, the `peerIP` and `asNumber` fields must be empty. | | [selector](#selector) | +| localWorkloadSelector | Selector for the local workloads that the node should peer with. When this is set, the `peerSelector` and `peerIP` fields must be empty and the `localWorkloadPeeringIPV4` and/or `localWorkloadPeeringIPV6` fields in the `BGPConfiguration` resource must be configured. It is also important to configure appropriate import/export filters when using this feature. See the [guide](../../networking/configuring/bgp-to-workload.mdx) for details. | | [selector](#selector) | | +| keepOriginalNextHop | Maintain and forward the original next hop BGP route attribute to a specific Peer within a different AS. | | boolean | +| extensions | Additional mapping of keys and values. Used for setting values in custom BGP configurations. | valid strings for both keys and values | map | | +| password | [BGP password](../../networking/configuring/secure-bgp.mdx) for the peerings generated by this BGPPeer resource. | | [BGPPassword](#bgppassword) | `nil` (no password) | +| sourceAddress | Specifies whether and how to configure a source address for the peerings generated by this BGPPeer resource. Default value "UseNodeIP" means to configure the node IP as the source address. "None" means not to configure a source address. | "UseNodeIP", "None" | string | "UseNodeIP" | +| failureDetectionMode | Specifies whether and how to detect loss of connectivity on the peerings generated by this BGPPeer resource. Default value "None" means nothing beyond BGP's own (slow) hold timer. "BFDIfDirectlyConnected" means to use BFD when the peer is directly connected. | "None", "BFDIfDirectlyConnected" | string | "None" | +| restartMode | Specifies restart behaviour to configure on the peerings generated by this BGPPeer resource. Default value "GracefulRestart" means traditional graceful restart. "LongLivedGracefulRestart" means LLGR according to draft-uttaro-idr-bgp-persistence-05. | "GracefulRestart", "LongLivedGracefulRestart" | string | "GracefulRestart" | +| maxRestartTime | Restart time that is announced by BIRD in the BGP graceful restart capability and that specifies how long the neighbor would wait for the BGP session to re-establish after a restart before deleting stale routes. When specified, this is configured as the graceful restart timeout when `RestartMode` is "GracefulRestart", and as the LLGR stale time when `RestartMode` is "LongLivedGracefulRestart". When not specified, the BIRD defaults are used, which are 120s for "GracefulRestart" and 3600s for "LongLivedGracefulRestart". Note: extra care should be taken when changing this configuration, as it may break networking in your cluster. | | duration | None | +| birdGatewayMode | Specifies the BIRD "gateway" mode, i.e. method for computing the immediate next hop for each received route, for peerings generated by this BGPPeer resource. Default value "Recursive" means "gateway recursive". "DirectIfDirectlyConnected" means to configure "gateway direct" when the peer is directly connected. | "Recursive", "DirectIfDirectlyConnected" | string | "Recursive" | +| numAllowedLocalASNumbers | The number of local AS numbers to allow in the AS path for received routes. This disables BGP loop prevention and should only be used if necessary. | | integer | `nil` (BIRD will default to 0 meaning no change to loop prevention behavior) | +| ttlSecurity | Enables the generalized TTL security mechanism (GTSM) which protects against spoofed packets by ignoring received packets with a smaller than expected TTL value. The provided value is the number of hops (edges) between the peers. | 0 - 255 | 8-bit integer | `nil` (results in BIRD configuration `ttl security off`) | +| filters | List of names of [BGPFilter](bgpfilter.mdx) resources to apply to this peering. | ["my-bgp-filter-1","my-bgp-filter-2"] | List of strings | | +| externalNetwork | Name of the external network to which this peer belongs. | - | string | | +| reachableBy | Adds a static route that may be needed to connect to a peer. In some cases, not having a static route for BGP peering results in route flapping. By adding the address of the gateway that the peer is connected to, a static route is added to prevent route flapping. | The address of the gateway that the peer is connected to | string | | +| nextHopMode | Specifies the method of calculating the next hop attribute for received routes. This replaces and expands the deprecated KeepOriginalNextHop field. Users should use this setting to control the next hop attribute for a BGP peer. When this is set, the value of the keepOriginalNextHop field is ignored. If neither keepOriginalNextHop or nextHopMode is specified, BGP's default behavior is used. Default value “Auto” means to apply BGP’s default behavior. "Self" means to configure "next hop self;" in "bird.cfg". "Keep" means to configure "next hop keep;" in "bird.cfg". | "Auto", "Self", "Keep" | string | "Auto" | +| reversePeering | Specifies whether Calico automatically generates reverse peerings for nodes selected by the peerSelector. If set to Manual, a separate BGPPeer must be created for the reverse peering. | "Auto", "Manual" | string | "Auto" | + +:::tip + +The cluster-wide default local AS number used when speaking with a peer is controlled by the +[BGPConfiguration resource](bgpconfig.mdx). That value can be overridden per-node by using the `bgp` field of +the [node resource](node.mdx). + +::: + +### BGPPassword + +:::note + +BGP passwords must be 80 characters or fewer. If a password longer than that +is configured, the BGP sessions with that password will fail to be established. + +::: + +| Field | Description | Schema | +| ------------ | ------------------------------- | ----------------- | +| secretKeyRef | Get the password from a secret. | [KeyRef](#keyref) | + +### KeyRef + +KeyRef tells $[prodname] where to get a BGP password. The referenced Kubernetes +secret must be in the same namespace as the $[nodecontainer] pod. + +| Field | Description | Schema | +| ----- | ------------------------- | ------ | +| name | The name of the secret | string | +| key | The key within the secret | string | + +:::warning + +$[prodname] must be able to read the referenced secret. + +This means that the `calico-node` ServiceAccount must have permissions to `list, get, watch` the secret referenced in the KeyRef. + +In practice, this can be done by creating a Role (which allows to `list, get, watch` the secret) and a RoleBinding (which grants the Role's permission to the `calico-node` ServiceAccount). + +Example: + +```yaml +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: bgp-passwords-reader + namespace: calico-system +rules: + - apiGroups: [""] + resources: ["secrets"] + resourceNames: ["bgp-passwords"] + verbs: ["list", "watch", "get"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: calico-read-bgp-passwords + namespace: calico-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: bgp-passwords-reader +subjects: + - kind: ServiceAccount + name: calico-node + +--- +apiVersion: v1 +kind: Secret +metadata: + name: "bgp-passwords" + namespace: calico-system +data: + peer_a_pw: "base64-encoded Password for Peer A" + peer_b_pw: "base64-encoded Password for Peer B" + +--- +apiVersion: projectcalico.org/v3 +kind: BGPPeer +metadata: + name: "peer-a" +spec: + password: + secretKeyRef: + name: "bgp-passwords" + key: "peer_a_pw" + +--- +apiVersion: projectcalico.org/v3 +kind: BGPPeer +metadata: + name: "peer-b" +spec: + password: + secretKeyRef: + name: "bgp-passwords" + key: "peer_b_pw" +``` + +::: + +## Peer scopes + +BGP Peers can exist at either global or node-specific scope. A peer's scope +determines which `$[nodecontainer]`s will attempt to establish a BGP session with that peer. +If `$[nodecontainer]` has a `listenPort` set in `BGPConfiguration`, it will be used in peering. + +### Global peer + +To assign a BGP peer a global scope, omit the `node` and `nodeSelector` fields. All nodes in +the cluster will attempt to establish BGP connections with it + +### Node-specific peer + +A BGP peer can also be node-specific. When the `node` field is included, only the specified node +will peer with it. When the `nodeSelector` field is included, the nodes with labels that match that selector +will peer with it. + +## Supported operations + +| Datastore type | Create/Delete | Update | Get/List | Notes | +| --------------------- | ------------- | ------ | -------- | ----- | +| Kubernetes API server | Yes | Yes | Yes | + +## Selector + + + +[parse-duration]: https://golang.org/pkg/time/#ParseDuration diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/blockaffinity.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/blockaffinity.mdx new file mode 100644 index 0000000000..e44acdc384 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/blockaffinity.mdx @@ -0,0 +1,32 @@ +--- +description: Reference for the BlockAffinity resource in Calico Cloud connected clusters that records which node owns each IP address management block. +--- + +# Block affinity + +A block affinity resource (`BlockAffinity`) represents the affinity between a node and an IP address management (IPAM) block. +$[prodname] IPAM creates and manages these resources automatically; you don't normally need to create or modify them. + +## Block affinity definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | ------------------------------------------------------------------ | --------------------------------------------------- | ------ | +| name | Unique name to describe this resource instance. Must be specified. | Alphanumeric string with optional `.`, `_`, or `-`. | string | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ------- | -------------------------------------------------------------------------- | ----------------------------------- | ------- | ------- | +| state | State of the affinity with regard to any referenced IPAM blocks. | confirmed, pending, pendingDeletion | string | | +| node | The node that this affinity is assigned to. | The hostname of the node | string | | +| cidr | The CIDR range this block affinity references. | A valid IPv4 or IPv6 CIDR. | string | | +| deleted | When set to true, clients should treat this block as if it does not exist. | true, false | boolean | `false` | + +## Supported operations + +| Datastore type | Create | Delete | Update | Get/List | Watch | +| --------------------- | ------ | ------ | ------ | -------- | ----- | +| etcdv3 | No | No | No | Yes | Yes | +| Kubernetes API server | No | No | No | Yes | Yes | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/caliconodestatus.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/caliconodestatus.mdx new file mode 100644 index 0000000000..4a96ee0fbe --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/caliconodestatus.mdx @@ -0,0 +1,217 @@ +--- +description: Reference for the CalicoNodeStatus resource in Calico Cloud connected clusters that exposes per-node agent, BGP, and routing state. +--- + +# Calico node status + +A Calico node status resource (`CalicoNodeStatus`) reports status information for a node, for use during troubleshooting. +You create a `CalicoNodeStatus` resource for the node you want to inspect, and $[prodname] periodically writes the collected information to the resource's `status` field. + +Status of Border Gateway Protocol (BGP) agents, BGP sessions, and routes exposed to BGP agents is collected from Linux nodes only. **Windows nodes are not supported at this time.** +The resource is valid only when $[prodname] BGP networking is in use. + +## Notes + +Updating `CalicoNodeStatus` has a small performance impact on CPU and memory usage of the node, and adds load to the Kubernetes API server. + +In our testing on a ten node, full mesh cluster, a `CalicoNodeStatus` resource was created for each node where the update interval was set to ten seconds. On each node, this resulted in an increase in CPU use of 5% of a vCPU and an increase of 4MB of memory. The control plane node recorded an increase in CPU usage of 5% of a vCPU for these 10 nodes. + +:::caution + +The implementation of `CalicoNodeStatus` is designed to handle a small number of nodes (less than 10 is recommended) reporting back status in the same time. If `CalicoNodeStatus` are created for a large number of nodes, and with short update interval, +the kubernetes apiserver may become slower and less responsive. +You should create `CalicoNodeStatus` for the node you are interested in and for debugging purpose only. `CalicoNodeStatus` resource should be deleted upon the completion of the debugging process. + +::: + +## Sample YAML + +To use this function, the user creates a CalicoNodeStatus object for the node, specifying the information to collect and the interval it should be collected at. This example collects information for node "my-kadm-node-0" with an update interval of 10 seconds. + +```bash +kubectl apply -f -< diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/egressgatewaypolicy.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/egressgatewaypolicy.mdx new file mode 100644 index 0000000000..d9dcc0ee1e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/egressgatewaypolicy.mdx @@ -0,0 +1,83 @@ +--- +description: Reference for the EgressGatewayPolicy resource in Calico Cloud connected clusters that selects which pods route through which egress gateways. +--- + +# Egress gateway policy + +An EgressGatewayPolicy resource (`EgressGatewayPolicy`) represents a way to select +different egress gateways or skip one for different destinations. + +Rules in an Egress EgressGatewayPolicy are checked in Longest Prefix Match(LPM) fashion +like routers. As such it is not valid to use the exact destination in two rules. + +In order for an EgressGatewayPolicy to be used, its `name` must be added +to a pod or namespace by using `egress.projectcalico.org/egressGatewayPolicy` annotation. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: EgressGatewayPolicy +metadata: + name: my-egwpolicy +spec: + rules: + - destination: + cidr: 10.0.0.0/8 + description: "Local: no gateway" + - destination: + cidr: 11.0.0.0/8 + description: "Gateway to on prem" + gateway: + namespaceSelector: "projectcalico.org/name == 'default'" + selector: "egress-code == 'blue'" + maxNextHops: 2 + - description: "Gateway to internet" + gateway: + namespaceSelector: "projectcalico.org/name == 'default'" + selector: "egress-code == 'red'" + gatewayPreference: PreferNodeLocal +``` + +## Egress gateway policy definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | ------------------------------------------------------------------ | --------------------------------------------------- | ------ | +| name | Unique name to describe this resource instance. Must be specified. | Alphanumeric string with optional `.`, `_`, or `-`. | string | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| -------- | -------------------------------------- | --------------- | --------------------------------------------------------- | ------- | +| rules | List of egress gateway policies | | [Egress Gateway Policy Rule](#egress-gateway-policy-rule) | | + +### Egress gateway policy rule + +| Field | Description | Accepted Values | Schema | Default | +| ----------------- | ------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------- | ------- | +| description | A description of rule | | string | | +| destination | CIDR representing a destination | | [destination](#destination) | | +| gateway | egress gateway to be used for a destination | | [gateway](#egress-gateway) | | +| gatewayPreference | Hints about egress gateway selection | `None` for using all available egress gateway replicas from the selected deployment, or `PreferNodeLocal` to use only egress gateway replicas on the same local node as the client pod or namespace if available, otherwise fall back to the default behaviour. | | 'None' | + +### Destination + +| Field | Description | Accepted Values | Schema | Default | +| ---------- | ------------------------------------------- | --------------------------- | --------------------------- | ------- | +| cidr | CIDR of destination network | | string | | + +### Egress gateway + +| Field | Description | Accepted Values | Schema | Default | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------- | --------------------------- | ------- | +| selector | selector to choose an egress gateway deployment | | string | | +| namespaceSelector | name space of egress gateway deployment | | string | | +| maxNextHops | Specifies the maximum number of egress gateway replicas from the selected deployment that a pod should depend on. Replicas will be chosen in a manner that attempts to balance load across the whole egress gateway replicaset. If unset, or set to "0", egress traffic will behave in the default manner (load balanced over all available gateways). | | string | | + +## Supported operations + +| Datastore type | Create/Delete | Update | Get/List | Notes | +| --------------------- | ------------- | ------ | -------- | ----- | +| Kubernetes API server | Yes | Yes | Yes | | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/felixconfig.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/felixconfig.mdx new file mode 100644 index 0000000000..1d61a451ae --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/felixconfig.mdx @@ -0,0 +1,435 @@ +--- +description: Reference for the FelixConfiguration resource in Calico Cloud connected clusters that controls Felix data plane behavior cluster-wide. +toc_max_heading_level: 4 +--- + +import FelixConfig from '../../_includes/components/FelixConfig'; + +# Felix configuration + +A Felix configuration resource (`FelixConfiguration`) represents Felix configuration options for the cluster. + +For `kubectl` commands, the following case-insensitive aliases may be used to specify the resource type on the CLI: `felixconfiguration.projectcalico.org`, `felixconfigurations.projectcalico.org` as well as abbreviations such as `felixconfiguration.p` and `felixconfigurations.p`. + +See [Configuring Felix](../component-resources/node/felix/configuration.mdx) for more details. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: FelixConfiguration +metadata: + name: default +spec: + ipv6Support: false + ipipMTU: 1400 + chainInsertMode: Append +``` + +## Felix configuration definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | --------------------------------------------------------- | --------------------------------------------------- | ------ | +| name | Unique name to describe this resource instance. Required. | Alphanumeric string with optional `.`, `_`, or `-`. | string | + +- $[prodname] automatically creates a resource named `default` containing the global default configuration settings for Felix. +{/*TODO-XREF-CC You can use [calicoctl](/reference/clis/calicoctl/overview/) to view and edit these settings*/} +- The resources with the name `node.` contain the node-specific overrides, and will be applied to the node ``. When deleting a node the FelixConfiguration resource associated with the node will also be deleted. +- Resources with any other name can use the `nodeSelector` field to target a group of nodes by label (see [Selector-scoped configuration](#selector-scoped-configuration) below). + +### Selector-scoped configuration + +:::note + +This feature is a **tech preview**. The behavior described here may change in future releases. + +::: + +FelixConfiguration resources whose name is not `default` and does not start with `node.` can include a `nodeSelector` field to restrict the configuration to nodes whose labels match the selector. This allows you to apply different Felix settings to groups of nodes without creating individual per-node resources. + +#### Example + +```yaml +apiVersion: projectcalico.org/v3 +kind: FelixConfiguration +metadata: + name: gpu-nodes +spec: + nodeSelector: "role == 'gpu'" + logSeverityScreen: Debug + bpfEnabled: true +``` + +This resource applies `logSeverityScreen: Debug` and `bpfEnabled: true` only to nodes with the label `role=gpu`. + +#### Configuration precedence + +When a selector-scoped FelixConfiguration matches a node, its settings are applied between per-node and global configuration in the priority order: + +1. Environment variables (highest priority) +2. Felix configuration file +3. Host-specific `FelixConfiguration` resource (`node.`) +4. **Selector-scoped `FelixConfiguration` resources** +5. Global `FelixConfiguration` resource (`default`) +6. Built-in defaults (lowest priority) + +A selector-scoped setting overrides the same setting from the global `default` resource, but is itself overridden by a per-node `node.` resource or environment variable. + +#### Overlapping selectors + +At most one selector-scoped FelixConfiguration should match any given node. If multiple selector-scoped resources match the same node, this is treated as a misconfiguration: the oldest resource by creation time is used and a warning is logged. This behavior may change in future releases. + +#### Restrictions + +- The `nodeSelector` field cannot be set on the `default` resource or on `node.` resources. These names have fixed scoping semantics. +- The `nodeSelector` field uses the same [selector syntax](../resources/networkpolicy.mdx#selector) as other Calico resources. + +### Spec + +#### Datastore connection + + + +#### Process: Feature detection/overrides + + + +#### Process: Go runtime + + + +#### Process: Health port and timeouts + + + +#### Process: Logging + + + +#### Process: Prometheus metrics + + + +#### Dataplane: Common + + + +#### Dataplane: iptables + + + +#### Dataplane: nftables + + + +#### Dataplane: eBPF + + + +#### Dataplane: Windows + + + +#### Dataplane: OpenStack support + + + +#### Dataplane: XDP acceleration for iptables dataplane + + + +#### Overlay: VXLAN overlay + + + +#### Overlay: IP-in-IP + + + +#### Overlay: Wireguard + + + +#### Overlay: IPSec + + + +#### Flow logs: Prometheus reports + + + +#### Flow logs: Syslog reports + + + +#### Flow logs: file reports + + + +#### DNS logs / policy + + + +#### L7 logs + + + +#### AWS integration + + + +#### Egress gateway + + + +#### External network support + + + +#### Packet capture + + + +#### L7 proxy + + +#### Debug/test-only (generally unsupported) + + + +#### Usage reporting + + + +### Health Timeout Overrides + +Felix has internal liveness and readiness watchdog timers that monitor its various loops. +If a loop fails to "check in" within the allotted timeout then Felix will report non-Ready +or non-Live on its health port (which is monitored by Kubelet in a Kubernetes system). +If Felix reports non-Live, this can result in the Pod being restarted. + +In Kubernetes, if you see the calico-node Pod readiness or liveness checks fail +intermittently, check the calico-node Pod log for a log from Felix that gives the +overall health status (the list of components will depend on which features are enabled): + +``` ++---------------------------+---------+----------------+-----------------+--------+ +| COMPONENT | TIMEOUT | LIVENESS | READINESS | DETAIL | ++---------------------------+---------+----------------+-----------------+--------+ +| CalculationGraph | 30s | reporting live | reporting ready | | +| FelixStartup | 0s | reporting live | reporting ready | | +| InternalDataplaneMainLoop | 1m30s | reporting live | reporting ready | | ++---------------------------+---------+----------------+-----------------+--------+ +``` + +If some health timeouts show as "timed out" it may help to apply an override +using the `healthTimeoutOverrides` field: + +```yaml noValidation +... +spec: + healthTimeoutOverrides: + - name: InternalDataplaneMainLoop + timeout: "5m" + - name: CalculationGraph + timeout: "1m30s" + ... +``` + +A timeout value of 0 disables the timeout. + +### ProtoPort + +| Field | Description | Accepted Values | Schema | +| -------- | -------------------- | ------------------------------------ | ------ | +| port | The exact port match | 0-65535 | int | +| protocol | The protocol match | tcp, udp, sctp | string | +| net | The CIDR match | any valid CIDR (e.g. 192.168.0.0/16) | string | + +Keep in mind that in the following example, `net: ""` and `net: "0.0.0.0/0"` are processed as the same in the policy enforcement. + +```yaml noValidation + ... +spec: + failsafeInboundHostPorts: + - net: "192.168.1.1/32" + port: 22 + protocol: tcp + - net: "" + port: 67 + protocol: udp +failsafeOutboundHostPorts: + - net: "0.0.0.0/0" + port: 67 + protocol: udp + ... +``` + +### AggregationKind + +| Value | Description | +| ----- | ---------------------------------------------------------------------------------------- | +| 0 | No aggregation | +| 1 | Aggregate all flows that share a source port on each node | +| 2 | Aggregate all flows that share source ports or are from the same ReplicaSet on each node | + +### DNSPolicyMode + +| Value | Description | +| ----------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| DelayDeniedPacket (default) | Felix delays any denied packet that traversed a policy that included egress domain matches, but did not match. The packet is released after a fixed time, or after the destination IP address was programmed. | +| DelayDNSResponse | Felix delays any DNS response until related IPSets are programmed. This introduces some latency to all DNS packets (even when no IPSet programming is required), but it ensures policy hit statistics are accurate. This is the recommended setting when you are making use of staged policies or policy rule hit statistics. A Linux kernel version of 3.13 or greater is required to use `DelayDNSResponse`. For earlier kernel versions, this value is modified to `DelayDeniedPacket`. | +| Inline | Parses DNS response inline with DNS response packet processing within iptables. This guarantees the DNS rules reflect any change immediately. This mode works for iptables only and matches the same mode for `BPFDNSPolicyMode`. This setting is ignored on Windows and `NoDelay` is always used. | +| NoDelay | Felix does not introduce any delay to the packets. DNS rules may not have been programmed by the time the first packet traverses the policy rules. Client applications need to handle reconnection attempts if initial connection attempts fail. This may be problematic for some applications or for very low DNS TTLs. | + +On Windows, or when using the eBPF dataplane, this setting is ignored. Windows always uses `NoDelay` while eBPF has its own [BPFDNSPolicyMode](#bpfdnspolicymode) option. + +### BPFDNSPolicyMode + +| Value | Description | +| ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Inline | Felix does not introduce any delay to any packets. Felix's eBPF programs parse DNS responses and program policy rules immediately, before the DNS response is passed to the application. This only applies to wildcard prefixes: `*.x.y.z` will be processed in this manner, but wildcards such as `x.*.y.z` will not match. A Linux kernel version of 5.17 or greater is required. | +| NoDelay | Felix does not introduce any delay to the packets. DNS rules may not have been programmed by the time the first packet traverses the policy rules. Client applications need to handle reconnection attempts if initial connection attempts fail. This may be problematic for some applications or for very low DNS TTLs. | + +### RouteTableRange + +The `RouteTableRange` option is now deprecated in favor of [RouteTableRanges](#routetableranges). + +| Field | Description | Accepted Values | Schema | +| ----- | -------------------- | --------------- | ------ | +| min | Minimum index to use | 1-250 | int | +| max | Maximum index to use | 1-250 | int | + +### RouteTableRanges + +`RouteTableRanges` is a list of `RouteTableRange` objects: + +| Field | Description | Accepted Values | Schema | +| ----- | -------------------- | --------------- | ------ | +| min | Minimum index to use | 1 - 4294967295 | int | +| max | Maximum index to use | 1 - 4294967295 | int | + +Each item in the `RouteTableRanges` list designates a range of routing tables available to Calico. By default, Calico will use a single range of `1-250`. If a range spans Linux's reserved table range (`253-255`) then those tables are automatically excluded from the list. It's possible that other table ranges may also be reserved by third-party systems unknown to Calico. In that case, multiple ranges can be defined to target tables below and above the sensitive ranges: + +```sh + target tables 65-99, and 256-1000, skipping 100-255 +calicoctl patch felixconfig default --type=merge -p '{"spec":{"routeTableRanges": [{"min": 65, "max": 99}, {"min": 256, "max": 1000}] }} +``` + +_Note_, for performance reasons, the maximum total number of routing tables that Felix will accept is 65535 (or 2\*16). + +Specifying both the `RouteTableRange` and `RouteTableRanges` arguments is not supported and will result in an error from the api. + +### AWS IAM Role/Policy for source-destination-check configuration + +Setting `awsSrcDstCheck` to `Disable` will automatically disable source-destination-check on EC2 instances in a cluster, provided necessary IAM roles and policies are set. One of the policies assigned to IAM role of cluster nodes must contain a statement similar to the following: + +``` +{ + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:ModifyNetworkInterfaceAttribute" + ], + "Resource": "*" +} +``` + +If there are no policies attached to node roles containing the above statement, attach a new policy. For example, if a node role is `test-cluster-nodeinstance-role`, click on the IAM role in AWS console. In the `Permission policies` list, add a new inline policy with the above statement to the new policy JSON definition. For detailed information, see [AWS documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html?icmpid=docs_iam_console). + +For an EKS cluster, the necessary IAM role and policy is available by default. No further actions are needed. + +## Supported operations + +| Datastore type | Create | Delete | Delete (Global `default`) | Update | Get/List | Notes | +| --------------------- | ------ | ------ | ------------------------- | ------ | -------- | ----- | +| Kubernetes API server | Yes | Yes | No | Yes | Yes | | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/globalalert.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/globalalert.mdx new file mode 100644 index 0000000000..e98f9a984e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/globalalert.mdx @@ -0,0 +1,317 @@ +--- +description: Reference for the GlobalAlert resource in Calico Cloud connected clusters that defines an alerting query against flow, audit, or DNS logs. +--- + +# Global Alert + +A global alert resource represents a query that is periodically run +against data sets collected by $[prodname] whose findings are +added to the Alerts page in the $[prodname] web console. Alerts may +search for the existence of rows in a query, or when aggregated metrics +satisfy a condition. + +$[prodname] supports alerts on the following data sets: + +- [Audit logs](../../observability/elastic/audit-overview.mdx) +- [DNS logs](../../observability/elastic/dns/index.mdx) +- [Flow logs](../../observability/elastic/flow/index.mdx) +- [L7 logs](../../observability/elastic/l7/index.mdx) + +For `kubectl` [commands](https://kubernetes.io/docs/reference/kubectl/overview/), the following case-insensitive aliases +can be used to specify the resource type on the CLI: +`globalalert.projectcalico.org`, `globalalerts.projectcalico.org` and abbreviations such as +`globalalert.p` and `globalalerts.p`. + +## Sample YAML + +```yaml noValidation +apiVersion: projectcalico.org/v3 +kind: GlobalAlert +metadata: + name: sample +spec: + summary: 'Sample' + description: 'Sample ${source_namespace}/${source_name_aggr}' + severity: 100 + dataSet: flows + query: action=allow + aggregateBy: [source_namespace, source_name_aggr] + field: num_flows + metric: sum + condition: gt + threshold: 0 +``` + +## GlobalAlert definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | ----------------------- | ----------------------------------------- | ------ | +| name | The name of this alert. | Lower-case alphanumeric with optional `-` | string | + +### Spec + +| Field | Description | Type | Required | Acceptable Values | Default | +| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------- | ----------------------------------------- | ---------------------------- | -------------------------------- | +| type | Type will dictate how the fields of the GlobalAlert will be utilized. Each `type` will have different usages and/or defaults for the other GlobalAlert fields as described in the table. | string | no | RuleBased | RuleBased | +| description | Human-readable description of the template. | string | yes | +| summary | Template for the description field in generated events. See the summary section below for more details. `description` is used if this is omitted. | string | no | +| severity | Severity of the alert for display in Manager. | int | yes | 1 - 100 | +| dataSet | Which data set to execute the alert against. | string | if `type` is `RuleBased` | audit, dns, flows, l7, vulnerability | +| period | How often the query defined will run, if `type` is `RuleBased`. | duration | no | 1h 2m 3s | 5m, 15m if `type` is `RuleBased` | +| lookback | Specifies how far back in time data is to be collected. Must exceed audit log flush interval, `dnsLogsFlushInterval`, or `flowLogsFlushInterval` as appropriate. | duration | no | 1h 2m 3s | 10m | +| query | Which data to include from the source data set. Written in a domain-specific query language. See the query section below. | string | no | +| aggregateBy | An optional list of fields to aggregate results. | string array | no | +| field | Which field to aggregate results by if using a metric other than count. | string | if metric is one of avg, max, min, or sum | +| metric | A metric to apply to aggregated results. `count` is the number of log entries matching the aggregation pattern. Others are applied only to numeric fields in the logs. | string | no | avg, max, min, sum, count | +| condition | Compare the value of the metric to the threshold using this condition. | string | if metric defined | eq, not_eq, lt, lte, gt, gte | +| threshold | A numeric value to compare the value of the metric against. | float | if metric defined | +| substitutions | An optional list of values to replace variable names in query. | List of [GlobalAlertSubstitution](#globalalertsubstitution) | no | + +### GlobalAlertSubstitution + +| Field | Description | Type | Required | +| ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ | -------- | +| name | The name of the global alert substitution. It will be referenced by the variable names in query. Duplicate names are not allowed in the substitutions list. | string | yes | +| values | A list of values for this substitution. Wildcard operators asterisk (`*`) and question mark (`?`) are supported. | string array | yes | + +### Status + +| Field | Description | +| --------------- | ------------------------------------------------------------------------------------------- | +| lastUpdate | When the alert was last modified on the backend. | +| active | Whether the alert is active on the backend. | +| healthy | Whether the alert is in an error state or not. | +| lastExecuted | When the query for the alert last ran. | +| lastEvent | When the condition of the alert was last satisfied and an alert was successfully generated. | +| errorConditions | List of errors preventing operation of the updates or search. | + +## Query + +Alerts use a domain-specific query language to select which records +from the data set should be used in the alert. This could be used to +identify flows with specific features, or to select (or omit) certain +namespaces from consideration. + +The query language is composed of any number of selectors, combined +with boolean expressions (`AND`, `OR`, and `NOT`), set expressions +(`IN` and `NOTIN`) and bracketed subexpressions. These are translated +by $[prodname] to Elastic DSL queries that are executed on the backend. + +Set expressions support wildcard operators asterisk (`*`) and question mark (`?`). +The asterisk sign matches zero or more characters and the question mark matches a single character. +Set values can be embedded into the query string or reference the values +in the global alert substitution list. + +A selector consists of a key, comparator, and value. Keys and values +may be identifiers consisting of alphanumerics and underscores (`_`) +with the first character being alphabetic or an underscore, or may be +quoted strings. Values may also be integer or floating point numbers. +Comparators may be `=` (equal), `!=` (not equal), `<` (less than), +`<=` (less than or equal), `>` (greater than), or `>=` (greater than +or equal). + +Keys must be indexed fields in their corresponding data set. See the +appendix for a list of valid keys in each data set. + +Examples: + +- `query: "count > 0"` +- `query: "\"servers.ip\" = \"127.0.0.1\""` + +Selectors may be combined using `AND`, `OR`, and `NOT` boolean expressions, +`IN` and `NOTIN` set expressions, and bracketed subexpressions. + +Examples: + +- `query: "count > 100 AND client_name=mypod"` +- `query: "client_namespace = ns1 OR client_namespace = ns2"` +- `query: "count > 100 AND NOT (client_namespace = ns1 OR client_namespace = ns2)"` +- `query: "(qtype = A OR qtype = AAAA) AND rcode != NoError"` +- `query: "process_name IN {\"proc1?\", \"*proc2\"} AND source_namespace = ns1` +- `query: "qname NOTIN ${domains}"` + +## Aggregation + +Results from the query can be aggregated by any number of data fields. +Only these data fields will be included in the generated alerts, and +each unique combination of aggregations will generate a unique alert. +Careful consideration of fields for aggregation will yield the best +results. + +Some good choices for aggregations on the `flows` data set are +`[source_namespace, source_name_aggr, source_name]`, `[source_ip]`, +`[dest_namespace, dest_name_aggr, dest_name]`, and `[dest_ip]` +depending on your use case. For the `dns` data set, +`[client_namespace, client_name_aggr, client_name]` is a good choice +for an aggregation pattern. + +## Metrics and conditions + +Results from the query can be further aggregated using a metric that +is applied to a numeric field, or counts the number of rows in an +aggregation. Search hits satisfying the condition are output as +alerts. + +| Metric | Description | Applied to Field | +| ------ | ---------------------------------- | ---------------- | +| count | Counts the number of rows | No | +| min | The minimal value of the field | Yes | +| max | The maximal value of the field | Yes | +| sum | The sum of all values of the field | Yes | +| avg | The average value of the field | Yes | + +| Condition | Description | +| --------- | --------------------- | +| eq | Equals | +| not_eq | Not equals | +| lt | Less than | +| lte | Less than or equal | +| gt | Greater than | +| gte | Greater than or equal | + +Example: + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalAlert +metadata: + name: frequent-dns-responses +spec: + description: 'Monitor for NXDomain' + summary: 'Observed ${sum} NXDomain responses for ${qname}' + severity: 100 + dataSet: dns + query: rcode = NXDomain AND (rtype = A or rtype = AAAA) + aggregateBy: qname + field: count + metric: sum + condition: gte + threshold: 100 +``` + +This alert identifies non-existing DNS responses for Internet addresses +that were observed more than 100 times in the past 10 minutes. + +### Unconditional alerts + +If the `field`, `metric`, `condition`, and `threshold` fields of an +alert are left blank then the alert will trigger whenever its query +returns any data. Each hit (or aggregation pattern, if `aggregateBy` +is non-empty) returned will cause an event to be created. This should +be used **only** when the query is highly specific to avoid filling +the Alerts page and index with a large number of events. The use of +`aggregateBy` is strongly recommended to reduce the number of entries +added to the Alerts page. + +The following example would alert on incoming connections to postgres +pods from the Internet that were not denied by policy. It runs hourly +to reduce the noise. Noise could be further reduced by removing +`source_ip` from the `aggregateBy` clause at the cost of removing +`source_ip` from the generated events. + +```yaml +period: 1h +lookback: 75m +query: 'dest_labels="application=postgres" AND source_type=net AND action=allow AND proto=tcp AND dest_port=5432' +aggregateBy: [dest_namespace, dest_name, source_ip] +``` + +## Summary template + +Alerts may include a summary template to provide context for the +alerts in the $[prodname] web console Alert user interface. Any field +in the `aggregateBy` section, or the value of the `metric` may be +substituted in the summary using a bracketed variable syntax. + +Example: + +```yaml +summary: 'Observed ${sum} NXDomain responses for ${qname}' +``` + +The `description` field is validated in the same manner. If not +provided, the `description` field is used in place of the `summary` +field. + +## Period and lookback + +The interval between alerts, and the amount of data considered by the +alert may be controlled using the `period` and `lookback` parameters +respectively. These fields are formatted as [duration](https://golang.org/pkg/time/#ParseDuration) strings. + +> A duration string is a possibly signed sequence of decimal numbers, +> each with optional fraction and a unit suffix, such as "300ms", +> "-1.5h" or "2h45m". Valid time units are "ns", "us" (or "µs"), +> "ms", "s", "m", "h". + +The default period is 5 minutes, and lookback is 10 minutes. The lookback +should always be greater than the sum of the period and the configured +`FlowLogsFlushInterval` or `DNSLogsFlushInterval` as appropriate to avoid gaps +in coverage. + +## Alert records + +With only aggregations and no metrics, the alert will generate one event +per aggregation pattern returned by the query. The record field will +contain only the aggregated fields. As before, this should be used +with specific queries. + +The addition of a metric will include the value of that metric in the +record, along with any aggregations. This, combined with queries as +necessary, will yield the best results in most cases. + +With no aggregations the alert will generate one event per record +returned by the query. The record will be included in its entirety +in the record field of the event. This should only be used with very +narrow and specific queries. + +## Templates + +$[prodname] supports the `GlobalAlertTemplate` resource type. +These are used in the $[prodname] web console to create alerts +with prepopulated fields that can be modified to suit your needs. +The `GlobalAlertTemplate` resource is configured identically to the +`GlobalAlert` resource. $[prodname] includes some sample Alert +templates; add your own templates as needed. + +### Sample YAML + +**RuleBased GlobalAlert** + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalAlertTemplate +metadata: + name: http.connections +spec: + description: 'HTTP connections to a target namespace' + summary: 'HTTP connections from ${source_namespace}/${source_name_aggr} to /${dest_name_aggr}' + severity: 50 + dataSet: flows + query: dest_namespace="" AND dest_port=80 + aggregateBy: [source_namespace, dest_name_aggr, source_name_aggr] + field: count + metric: sum + condition: gte + threshold: 1 +``` + +## Appendix: Valid fields for queries + +### Audit logs + +See [audit.k8s.io group v1](https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apiserver/pkg/apis/audit/v1/types.go) for descriptions of fields. + +### DNS logs + +See [DNS logs](../../observability/elastic/dns/dns-logs.mdx) for description of fields. + +### Flow logs + +See [Flow logs](../../observability/elastic/flow/datatypes.mdx) for description of fields. + +### L7 logs + +See [L7 logs](../../observability/elastic/l7/datatypes.mdx) for description of fields. diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/globalnetworkpolicy.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/globalnetworkpolicy.mdx new file mode 100644 index 0000000000..11721593d4 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/globalnetworkpolicy.mdx @@ -0,0 +1,213 @@ +--- +description: Reference for the GlobalNetworkPolicy resource in Calico Cloud, a cluster-scoped tiered policy that selects endpoints across all namespaces in a connected cluster. +--- + +# Global network policy + +import Servicematch from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_servicematch.mdx'; + +import Serviceaccountmatch from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_serviceaccountmatch.mdx'; + +import Ports from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_ports.mdx'; + +import SelectorScopes from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_selector-scopes.mdx'; + +import Selectors from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_selectors.mdx'; + +import Entityrule from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_entityrule.mdx'; + +import Icmp from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_icmp.mdx'; + +import Rule from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_rule.mdx'; + +import Httpmatch from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_httpmatch.mdx'; + + +A global network policy resource (`GlobalNetworkPolicy`) represents an ordered set of rules which are applied +to a collection of endpoints that match a [label selector](#selector). + +`GlobalNetworkPolicy` is not a namespaced resource. `GlobalNetworkPolicy` applies to [workload endpoint resources](workloadendpoint.mdx) in all namespaces, and to [host endpoint resources](hostendpoint.mdx). +Select a namespace in a `GlobalNetworkPolicy` in the standard selector by using +`projectcalico.org/namespace` as the label name and a `namespace` name as the +value to compare against, e.g., `projectcalico.org/namespace == "default"`. +See [network policy resource](networkpolicy.mdx) for namespaced network policy. + +`GlobalNetworkPolicy` resources can be used to define network connectivity rules between groups of $[prodname] endpoints and host endpoints. +{/*TODO-XREF-CC +and take precedence over [Profile resources](/reference/resources/profile) if any are defined. +*/} + +GlobalNetworkPolicies are organized into [tiers](tier.mdx), which provide an additional layer of ordering—in particular, note that the `Pass` action skips to the +next [tier](tier.mdx), to enable hierarchical security policy. + +For `kubectl` [commands](https://kubernetes.io/docs/reference/kubectl/overview/), the following case-insensitive aliases +may be used to specify the resource type on the CLI: +`globalnetworkpolicy.projectcalico.org`, `globalnetworkpolicies.projectcalico.org` and abbreviations such as +`globalnetworkpolicy.p` and `globalnetworkpolicies.p`. + +## Sample YAML + +This sample policy allows TCP traffic from `frontend` endpoints to port 6379 on +`database` endpoints. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: internal-access.allow-tcp-6379 +spec: + tier: internal-access + selector: role == 'database' + types: + - Ingress + - Egress + ingress: + - action: Allow + metadata: + annotations: + from: frontend + to: database + protocol: TCP + source: + selector: role == 'frontend' + destination: + ports: + - 6379 + egress: + - action: Allow +``` + +## Definition + +### Metadata + +| Field | Description | Accepted Values | Schema | Default | +| ----- | ----------------------------------------- | --------------------------------------------------- | ------ | ------- | +| name | The name of the network policy. Required. | Alphanumeric string with optional `.`, `_`, or `-`. | string | | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------- | --------------------- | --------------------------------------------- | +| order | Controls the order of precedence. $[prodname] applies the policy with the lowest value first. | | float | | +| tier | Name of the [tier](tier.mdx) this policy belongs to. | | string | `default` | +| selector | Selects the endpoints to which this policy applies. | | [selector](#selector) | all() | +| serviceAccountSelector | Selects the service account(s) to which this policy applies. Select all service accounts in the cluster with a specific name using the `projectcalico.org/name` label. | | [selector](#selector) | all() | +| namespaceSelector | Selects the namespace(s) to which this policy applies. Select a specific namespace by name using the `projectcalico.org/name` label. | | [selector](#selector) | all() | +| types | Applies the policy based on the direction of the traffic. To apply the policy to inbound traffic, set to `Ingress`. To apply the policy to outbound traffic, set to `Egress`. To apply the policy to both, set to `Ingress, Egress`. | `Ingress`, `Egress` | List of strings | Depends on presence of ingress/egress rules\* | +| ingress | Ordered list of ingress rules applied by policy. | | List of [Rule](#rule) | | +| egress | Ordered list of egress rules applied by this policy. | | List of [Rule](#rule) | | +| doNotTrack\*\* | Indicates to apply the rules in this policy before any data plane connection tracking, and that packets allowed by these rules should not be tracked. | true, false | boolean | false | +| preDNAT\*\* | Indicates to apply the rules in this policy before any DNAT. | true, false | boolean | false | +| applyOnForward\*\* | Indicates to apply the rules in this policy on forwarded traffic as well as to locally terminated traffic. | true, false | boolean | false | +| performanceHints | Contains a list of hints to Calico's policy engine to help process the policy more efficiently. Hints never change the enforcement behaviour of the policy. The available hints are described [below](#performance-hints). | `AssumeNeededOnEveryNode` | List of strings | | + +\* If `types` has no value, $[prodname] defaults as follows. + +> | Ingress Rules Present | Egress Rules Present | `Types` value | +> | --------------------- | -------------------- | ----------------- | +> | No | No | `Ingress` | +> | Yes | No | `Ingress` | +> | No | Yes | `Egress` | +> | Yes | Yes | `Ingress, Egress` | + +\*\* The `doNotTrack` and `preDNAT` and `applyOnForward` fields are meaningful +only when applying policy to a [host endpoint](hostendpoint.mdx). + +Only one of `doNotTrack` and `preDNAT` may be set to `true` (in a given policy). If they are both `false`, or when applying the policy to a +[workload endpoint](workloadendpoint.mdx), +the policy is enforced after connection tracking and any DNAT. + +`applyOnForward` must be set to `true` if either `doNotTrack` or `preDNAT` is +`true` because for a given policy, any untracked rules or rules before DNAT will +in practice apply to forwarded traffic. + +### Rule + + + +### ICMP + + + +### EntityRule + + + +### Selector + + + + +### Ports + + + +### ServiceAccountMatch + + + +### ServiceMatch + + + +### Performance Hints + +Performance hints provide a way to tell $[prodname] about the intended use of the policy so that it may +process it more efficiently. Currently only one hint is defined: + +* `AssumeNeededOnEveryNode`: normally, $[prodname] only calculates a policy's rules and selectors on nodes where + the policy is actually in use (i.e. its selector matches a local endpoint). This saves work in most cases. + The `AssumeNeededOnEveryNode` hint tells $[prodname] to treat the policy as "in use" on *every* node. This is + useful for large policy sets that are known to apply to all (or nearly all) endpoints. It effectively "preloads" + the policy on every node so that there is less work to do when the first endpoint matching the policy shows up. + It also prevents work from being done to tear down the policy when the last endpoint is drained. + +## Application layer policy + +Application layer policy is an optional feature of $[prodname] and +[must be enabled](../../network-policy/application-layer-policies/alp.mdx) +to use the following match criteria. + +:::note + +Application layer policy match criteria are supported with the following restrictions. + +- Only ingress policy is supported. Egress policy must not contain any application layer policy match clauses. +- Rules must have the action `Allow` if they contain application layer policy match clauses. + +::: + +### HTTPMatch + + + +## Supported operations + +| Datastore type | Create/Delete | Update | Get/List | Notes | +| ------------------------ | ------------- | ------ | -------- | ----- | +| Kubernetes API datastore | Yes | Yes | Yes | + +#### List filtering on tiers + +List and watch operations may specify label selectors or field selectors to filter `GlobalNetworkPolicy` resources on tiers returned by the API server. +When no selector is specified, the API server returns all `GlobalNetworkPolicy` resources from all tiers that the user has access to. + +##### Field selector + +When using the field selector, supported operators are `=` and `==` + +The following example shows how to retrieve all `GlobalNetworkPolicy` resources in the default tier: + +```bash +kubectl get globalnetworkpolicy --field-selector spec.tier=default +``` + +##### Label selector + +When using the label selector, supported operators are `=`, `==` and `IN`. + +The following example shows how to retrieve all `GlobalNetworkPolicy` resources in the `default` and `net-sec` tiers: + +```bash +kubectl get globalnetworkpolicy -l 'projectcalico.org/tier in (default, net-sec)' \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/globalnetworkset.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/globalnetworkset.mdx new file mode 100644 index 0000000000..89daf16f12 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/globalnetworkset.mdx @@ -0,0 +1,83 @@ +--- +description: Reference for the GlobalNetworkSet resource in Calico Cloud connected clusters that defines a cluster-scoped set of CIDRs referenced by tiered network policy. +--- + +# Global network set + +import DomainNames from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_domain-names.mdx'; + +A global network set resource (`GlobalNetworkSet`) represents an arbitrary set of IP subnetworks/CIDRs, +allowing it to be matched by $[prodname] policy. Network sets are useful for applying policy to traffic +coming from (or going to) external, non-$[prodname], networks. + +GlobalNetworkSets can also include domain names, whose effect is to allow egress traffic to those +domain names, when the GlobalNetworkSet is matched by the destination selector of an egress rule +with action Allow. Domain names have no effect in ingress rules, or in a rule whose action is not +Allow. + +:::note + +$[prodname] implements policy for domain names by learning the +corresponding IPs from DNS, then programming rules to allow those IPs. This means that +if multiple domain names A, B and C all map to the same IP, and there is domain-based +policy to allow A, traffic to B and C will be allowed as well. + +::: + +The metadata for each network set includes a set of labels. When $[prodname] is calculating the set of +IPs that should match a source/destination selector within a +[global network policy](globalnetworkpolicy.mdx) rule, or within a +[network policy](networkpolicy.mdx) rule whose `namespaceSelector` includes `global()`, it includes +the CIDRs from any network sets that match the selector. + +:::note + +Since $[prodname] matches packets based on their source/destination IP addresses, +$[prodname] rules may not behave as expected if there is NAT between the $[prodname]-enabled node and the +networks listed in a network set. For example, in Kubernetes, incoming traffic via a service IP is +typically SNATed by the kube-proxy before reaching the destination host so $[prodname]'s workload +policy will see the kube-proxy's host's IP as the source instead of the real source. +For `kubectl` [commands](https://kubernetes.io/docs/reference/kubectl/overview/), the following case-insensitive aliases +may be used to specify the resource type on the CLI: +`globalnetworkset.projectcalico.org`, `globalnetworksets.projectcalico.org` and abbreviations such as +`globalnetworkset.p` and `globalnetworksets.p`. + +::: + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkSet +metadata: + name: a-name-for-the-set + labels: + role: external-database +spec: + nets: + - 198.51.100.0/28 + - 203.0.113.0/24 + allowedEgressDomains: + - db.com + - '*.db.com' +``` + +## Global network set definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ------ | ------------------------------------------ | ------------------------------------------------- | ------ | +| name | The name of this network set. | Lower-case alphanumeric with optional `-` or `-`. | string | +| labels | A set of labels to apply to this endpoint. | | map | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------- | ------ | ------- | +| nets | The IP networks/CIDRs to include in the set. | Valid IPv4 or IPv6 CIDRs, for example "192.0.2.128/25" | list | | +| allowedEgressDomains | The list of domain names that belong to this set and are honored in egress allow rules only. Domain names specified here only work to allow egress traffic from the cluster to external destinations. They don't work to _deny_ traffic to destinations specified by domain name, or to allow ingress traffic from _sources_ specified by domain name. | List of [exact or wildcard domain names](#exact-and-wildcard-domain-names) | list | | + +### Exact and wildcard domain names + + diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/globalthreatfeed.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/globalthreatfeed.mdx new file mode 100644 index 0000000000..a826600827 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/globalthreatfeed.mdx @@ -0,0 +1,269 @@ +--- +description: Reference for the GlobalThreatFeed resource in Calico Cloud connected clusters that pulls indicators of compromise into Calico-managed network sets. +--- + +# Global threat feed + +A global threat feed resource (GlobalThreatFeed) represents a feed of threat intelligence used for +security purposes. + +$[prodname] supports threat feeds that give either + +- a set of IP addresses or IP prefixes, with content type IPSet, or +- a set of domain names, with content type DomainNameSet + +For each IPSet threat feed, $[prodname] automatically monitors flow logs for members of the set. +IPSet threat feeds can also be configured to be synchronized to a [global network set](globalnetworkset.mdx), +allowing you to use them as a dynamically-updating deny-list by incorporating the global network set into network policy. + +For each DomainNameSet threat feed, $[prodname] automatically monitors DNS logs for queries (QNAME) or answers (RR NAME or RDATA) that contain members of the set. + +For `kubectl` [commands](https://kubernetes.io/docs/reference/kubectl/overview/), the following case-insensitive aliases +may be used to specify the resource type on the CLI: +`globalthreatfeed.projectcalico.org`, `globalthreatfeeds.projectcalico.org` and abbreviations such as +`globalthreatfeed.p` and `globalthreatfeeds.p`. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalThreatFeed +metadata: + name: sample-global-threat-feed +spec: + content: IPSet + mode: Enabled + description: "This is the sample global threat feed" + feedType: Custom + globalNetworkSet: + # labels to set on the GNS + labels: + level: high + pull: + # accepts time in golang duration format + period: 24h + http: + format: + newlineDelimited: {} + url: https://an.example.threat.feed/deny-list + headers: + - name: "Accept" + value: "text/plain" + - name: "APIKey" + valueFrom: + # secrets selected must be in the "tigera-intrusion-detection" namespace to be used + secretKeyRef: + name: "globalthreatfeed-sample-global-threat-feed-example" + key: "apikey" +``` + +## Push or Pull + +You can configure $[prodname] to pull updates from your threat feed using a [`pull`](#pull) stanza in +the global threat feed spec. + +Alternately, you can have your threat feed push updates directly. Leave out the `pull` stanza, and configure +your threat feed to create or update the Elasticsearch document that corresponds to the global threat +feed object. + +For IPSet threat feeds, this Elasticsearch document will be in the index `.tigera.ipset.` and must have the ID set +to the name of the global threat feed object. The doc should have a single field called `ips`, containing +a list of IP prefixes. + +For example: + +``` +PUT .tigera.ipset.cluster01/_doc/sample-global-threat-feed +{ + "ips" : ["99.99.99.99/32", "100.100.100.0/24"] +} +``` + +For DomainNameSet threat feeds, this Elasticsearch document will be in the index `.tigera.domainnameset.` and must +have the ID set to the name of the global threat feed object. The doc should have a single field called `domains`, containing +a list of domain names. + +For example: + +``` +PUT .tigera.domainnameset.cluster01/_doc/example-global-threat-feed +{ + "domains" : ["malware.badstuff", "hackers.r.us"] +} +``` + +Refer to the [Elasticsearch document APIs][elastic-document-apis] for more information on how to +create and update documents in Elasticsearch. + +## GlobalThreatFeed Definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ------ | --------------------------------------------- | ----------------------------------------- | ------ | +| name | The name of this threat feed. | Lower-case alphanumeric with optional `-` | string | +| labels | A set of labels to apply to this threat feed. | | map | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ---------------- | ---------------------------------------------------- | ---------------------- | --------------------------------------------- | ------- | +| content | What kind of threat intelligence is provided | IPSet, DomainNameSet | string | IPSet | +| mode | Determines if the threat feed is Enabled or Disabled | Enabled, Disabled | string | Enabled | +| description | Human-readable description of the template | Maximum 256 characters | string | | +| feedType | Distinguishes Builtin threat feeds from Custom feeds | Builtin, Custom | string | Custom | +| globalNetworkSet | Include to sync with a global network set | | [GlobalNetworkSetSync](#globalnetworksetsync) | | +| pull | Configure periodic pull of threat feed updates | | [Pull](#pull) | | + +### Status + +The `status` is read-only for users and updated by the `intrusion-detection-controller` component as +it processes global threat feeds. + +| Field | Description | +| -------------------- | -------------------------------------------------------------------------------- | +| lastSuccessfulSync | Timestamp of the last successful update to the threat intelligence from the feed | +| lastSuccessfulSearch | Timestamp of the last successful search of logs for threats | +| errorConditions | List of errors preventing operation of the updates or search | + +### GlobalNetworkSetSync + +When you include a `globalNetworkSet` stanza in a global threat feed, it triggers synchronization +with a [global network set](globalnetworkset.mdx). This global network set will have the name `threatfeed.` +where `` is the name of the global threat feed it is synced with. This is only supported for +threat feeds of type IPSet. + +:::note + +A `globalNetworkSet` stanza only works for `IPSet` threat feeds, and you must also include a `pull` stanza. + +::: + +| Field | Description | Accepted Values | Schema | +| ------ | --------------------------------------------------------- | --------------- | ------ | +| labels | A set of labels to apply to the synced global network set | | map | + +### Pull + +When you include a `pull` stanza in a global threat feed, it triggers a periodic pull of new data. On successful +pull and update to the data store, we update the `status.lastSuccessfulSync` timestamp. + +If you do not include a `pull` stanza, you must configure your system to [push](#push-or-pull) updates. + +| Field | Description | Accepted Values | Schema | Default | +| ------ | ------------------------------------- | --------------- | --------------------------------- | ------- | +| period | How often to pull an update | ≥ 5m | [Duration string][parse-duration] | 24h | +| http | Pull the update from an HTTP endpoint | | [HTTPPull](#httppull) | | + +### HTTPPull + +Pull updates from the threat feed by doing an HTTP GET against the given URL. + +| Field | Description | Accepted Values | Schema | +| ------- | --------------------------------------------------------- | --------------- | ------------------------- | +| format | Format of the data the threat feed returns | | [Format](#format) | +| url | The URL to query | | string | +| headers | List of additional HTTP Headers to include on the request | | [HTTPHeader](#httpheader) | + +IPSet threat feeds must contain IP addresses or IP prefixes. For example: + +``` + This is an IP Prefix +100.100.100.0/24 + This is an address +99.99.99.99 +``` + +DomainNameSet threat feeds must contain domain names. For example: + +``` + Suspicious domains +malware.badstuff +hackers.r.us +``` + +Internationalized domain names (IDNA) may be encoded either as Unicode in UTF-8 format, or as +ASCII-Compatible Encoding (ACE) according to [RFC 5890][idna]. + +### Format + +Several different feed formats are supported. The default, +`newlineDelimited`, expects a text file containing entries separated by +newline characters. It may also include comments prefixed by `#`. +`json` uses a [jsonpath] to extract the desired information from a +JSON document. `csv` extracts one column from CSV-formatted data. + +| Field | Description | Schema | +| ---------------- | --------------------------- | ------------- | +| newlineDelimited | Newline-delimited text file | Empty object | +| json | JSON object | [JSON](#json) | +| csv | CSV file | [CSV](#csv) | + +#### JSON + +| Field | Description | Schema | +| ----- | ----------------------------- | ------ | +| path | [jsonpath] to extract values. | string | + +Values can be extracted from the document using any [jsonpath] +expression, subject to the limitations mentioned below, that evaluates +to a list of strings. For example: `$.` is valid for `["a", "b", "c"]`, +and `$.a` is valid for `{"a": ["b", "c"]}`. + +:::caution + +No support for subexpressions and filters. Strings in +brackets must use double quotes. It cannot operate on JSON decoded +struct fields. + +::: + +#### CSV + +| Field | Description | Schema | +| --------------------------- | ------------------------------------------------------------------------- | ------ | ------ | +| fieldNum | Number of column containing values. Mutually exclusive with `fieldName`. | int | +| fieldName | Name of column containing values, requires `header: true`. | string | +| header | Whether or not the document contains a header row. | bool | +| columnDelimiter | An alternative delimiter character, such as ` | `. | string | +| commentDelimiter | Lines beginning with this character are skipped. `#` is common. | string | +| recordSize | The number of columns expected in the document. Auto detected if omitted. | int | +| disableRecordSizeValidation | Disable row size checking. Mutually exclusive with `recordSize`. | bool | + +### HTTPHeader + +| Field | Description | Schema | +| --------- | --------------------------------------------------------- | ------------------------------------- | +| name | Header name | string | +| value | Literal value | string | +| valueFrom | Include to retrieve the value from a config map or secret | [HTTPHeaderSource](#httpheadersource) | + +:::note + +You must include either `value` or `valueFrom`, but not both. + +::: + +### HTTPHeaderSource + +| Field | Description | Schema | +| --------------- | ------------------------------- | ----------------- | +| configMapKeyRef | Get the value from a config map | [KeyRef](#keyref) | +| secretKeyRef | Get the value from a secret | [KeyRef](#keyref) | + +### KeyRef + +KeyRef tells $[prodname] where to get the value for a header. The referenced Kubernetes object +(either a config map or a secret) must be in the `tigera-intrusion-detection` namespace. The referenced +Kubernetes object should have a name with following prefix format: `globalthreatfeed--`. + +| Field | Description | Accepted Values | Schema | Default | +| -------- | --------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- | ------ | ------- | +| name | The name of the config map or secret | | string | | +| key | The key within the config map or secret | | string | | +| optional | Whether the pull can proceed without the referenced value | If the referenced value does not exist, `true` means omit the header. `false` means abort the entire pull until it exists | bool | `false` | + +[elastic-document-apis]: https://www.elastic.co/guide/en/elasticsearch/reference/6.4/docs-update.html +[parse-duration]: https://golang.org/pkg/time/#ParseDuration +[idna]: https://tools.ietf.org/html/rfc5890 +[jsonpath]: https://goessner.net/articles/JsonPath/ diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/hostendpoint.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/hostendpoint.mdx new file mode 100644 index 0000000000..d77cff2f70 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/hostendpoint.mdx @@ -0,0 +1,121 @@ +--- +description: Reference for the HostEndpoint resource in Calico Cloud connected clusters that represents a host network interface for tiered policy enforcement. +--- + +# Host endpoint + +import Endpointport from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_endpointport.mdx'; + +A host endpoint resource (`HostEndpoint`) represents one or more real or virtual interfaces +attached to a host that is running $[prodname]. It enforces $[prodname] policy on +the traffic that is entering or leaving the host's default network namespace through those +interfaces. + +- A host endpoint with `interfaceName: *` represents _all_ of a host's real or virtual + interfaces. + +- A host endpoint for one specific real interface is configured by `interfaceName: `, + for example `interfaceName: eth0`, or by leaving `interfaceName` + empty and including one of the interface's IPs in `expectedIPs`. + +Each host endpoint may include a set of labels and list of profiles that $[prodname] +will use to apply +[policy](networkpolicy.mdx) +to the interface. If no profiles or labels are applied, $[prodname] will not apply +any policy. + +For `kubectl` [commands](https://kubernetes.io/docs/reference/kubectl/overview/), the following case-insensitive aliases +may be used to specify the resource type on the CLI: +`hostendpoint.projectcalico.org`, `hostendpoints.projectcalico.org` and abbreviations such as +`hostendpoint.p` and `hostendpoints.p`. + +**Default behavior of external traffic to/from host** + +If a host endpoint is created and network policy is not in place, the $[prodname] default is to deny traffic to/from that endpoint (except for traffic allowed by failsafe rules). +For a named host endpoint (i.e. a host endpoint representing a specific interface), $[prodname] blocks traffic only to/from the interface specified in the host endpoint. Traffic to/from other interfaces is ignored. + +:::note + +Host endpoints with `interfaceName: *` do not support [untracked policy](../../network-policy/extreme-traffic/high-connection-workloads.mdx). + +::: + +For a wildcard host endpoint (i.e. a host endpoint representing all of a host's interfaces), $[prodname] blocks traffic to/from _all_ interfaces on the host (except for traffic allowed by failsafe rules). + +However, profiles can be used in conjunction with host endpoints to modify default behavior of external traffic to/from the host in the absence of network policy. +$[prodname] provides a default profile resource named `projectcalico-default-allow` that consists of allow-all ingress and egress rules. +Host endpoints with the `projectcalico-default-allow` profile attached will have "allow-all" semantics instead of "deny-all" in the absence of policy. + +Note: If you have custom iptables rules, using host endpoints with allow-all rules (with no policies) will accept all traffic and therefore bypass those custom rules. + +:::note + +Auto host endpoints specify the `projectcalico-default-allow` profile so they behave similarly to pod workload endpoints. + +::: + +:::note + +When rendering security rules on other hosts, $[prodname] uses the +`expectedIPs` field to resolve label selectors to IP addresses. If the `expectedIPs` field +is omitted then security rules that use labels will fail to match this endpoint. + +::: + +**Host to local workload traffic**: Traffic from a host to its workload endpoints (e.g. Kubernetes pods) is always allowed, despite any policy in place. This ensures that `kubelet` liveness and readiness probes always work. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: HostEndpoint +metadata: + name: some.name + labels: + type: production +spec: + interfaceName: eth0 + node: myhost + expectedIPs: + - 192.168.0.1 + - 192.168.0.2 + profiles: + - profile1 + - profile2 + ports: + - name: some-port + port: 1234 + protocol: TCP + - name: another-port + port: 5432 + protocol: UDP +``` + +## Host endpoint definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ------ | ------------------------------------------ | --------------------------------------------------- | ------ | +| name | The name of this hostEndpoint. Required. | Alphanumeric string with optional `.`, `_`, or `-`. | string | +| labels | A set of labels to apply to this endpoint. | | map | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ------------- | -------------------------------------------------------------------------- | -------------------------- | -------------------------------------- | ------- | +| node | The name of the node where this HostEndpoint resides. | | string | +| interfaceName | Either `*` or the name of the specific interface on which to apply policy. | | string | +| expectedIPs | The expected IP addresses associated with the interface. | Valid IPv4 or IPv6 address | list | +| profiles | The list of profiles to apply to the endpoint. | | list | +| ports | List of named ports that this workload exposes. | | List of [EndpointPorts](#endpointport) | + +### EndpointPort + + + +## Supported operations + +| Datastore type | Create/Delete | Update | Get/List | Notes | +| --------------------- | ------------- | ------ | -------- | ----- | +| Kubernetes API server | Yes | Yes | Yes | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/index.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/index.mdx new file mode 100644 index 0000000000..2976314a16 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/index.mdx @@ -0,0 +1,11 @@ +--- +description: Reference index of the Calico Cloud API resources covering networking, observability, threat feeds, and tiered policy custom resources in connected clusters. +hide_table_of_contents: true +--- + +# Resource definitions + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/ipamconfig.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/ipamconfig.mdx new file mode 100644 index 0000000000..7834072427 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/ipamconfig.mdx @@ -0,0 +1,45 @@ +--- +description: Reference for the IP address management configuration resource in Calico Cloud connected clusters that sets cluster-wide IPAM options. +--- + +# IPAM configuration + +An IP address management (IPAM) configuration resource (`IPAMConfiguration`) represents global IPAM configuration options. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPAMConfiguration +metadata: + name: default +spec: + strictAffinity: false + maxBlocksPerHost: 4 + kubeVirtVMAddressPersistence: Disabled +``` + +## IPAM configuration definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | --------------------------------------------------------- | --------------- | ------ | +| name | Unique name to describe this resource instance. Required. | default | string | + +The resource is a singleton which must have the name `default`. + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ---------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | ------------------ | ------ | -------- | +| strictAffinity | When StrictAffinity is true, borrowing IP addresses is not allowed. | true, false | bool | false | +| maxBlocksPerHost | The max number of blocks that can be affine to each host. | 0 - max(int32) | int | 20 | +| kubeVirtVMAddressPersistence | Controls whether KubeVirt VMs retain persistent IP addresses across lifecycle events such as restart or migration. Requires KubeVirt. | Enabled, Disabled | string | Disabled | + +## Supported operations + +| Datastore type | Create | Delete | Update | Get/List | +| --------------------- | ------ | ------ | ------ | -------- | +| etcdv3 | Yes | Yes | Yes | Yes | +| Kubernetes API server | Yes | Yes | Yes | Yes | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/ippool.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/ippool.mdx new file mode 100644 index 0000000000..a3d8369f6a --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/ippool.mdx @@ -0,0 +1,154 @@ +--- +description: Reference for the IPPool resource in Calico Cloud connected clusters that defines CIDRs available for pod IP address allocation. +--- + +# IP pool + +import Selectors from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_selectors.mdx'; + +An IP pool resource (`IPPool`) represents a collection of IP addresses from which $[prodname] expects +endpoint IPs to be assigned. + +For `kubectl` commands, the following case-insensitive aliases may be used to specify the resource type on the CLI: `ippool.projectcalico.org`, `ippools.projectcalico.org` as well as abbreviations such as `ippool.p` and `ippools.p`. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPPool +metadata: + name: my.ippool-1 +spec: + cidr: 10.1.0.0/16 + ipipMode: CrossSubnet + natOutgoing: true + disabled: false + nodeSelector: all() + allowedUses: + - Workload + - Tunnel +``` + +## IP pool definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | ------------------------------------------- | --------------------------------------------------- | ------ | +| name | The name of this IPPool resource. Required. | Alphanumeric string with optional `.`, `_`, or `-`. | string | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ---------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------- | --------------------------------------------- | +| cidr | IP range to use for this pool. | A valid IPv4 or IPv6 CIDR. Subnet length must be at least big enough to fit a single block (by default `/26` for IPv4 or `/122` for IPv6). Must not overlap with the Link Local range `169.254.0.0/16` or `fe80::/10`. | string | | +| blockSize | The CIDR size of allocation blocks used by this pool. Blocks are allocated on demand to hosts and are used to aggregate routes. The value can only be set when the pool is created. | 20 to 32 (inclusive) for IPv4 and 116 to 128 (inclusive) for IPv6 | int | `26` for IPv4 pools and `122` for IPv6 pools. | +| ipipMode | The mode defining when IPIP will be used. Cannot be set at the same time as `vxlanMode`. | Always, CrossSubnet, Never | string | `Never` | +| vxlanMode | The mode defining when VXLAN will be used. Cannot be set at the same time as `ipipMode`. | Always, CrossSubnet, Never | string | `Never` | +| natOutgoing | When enabled, packets sent from $[prodname] networked containers in this pool to destinations outside of any Calico IP pools will be masqueraded. | true, false | boolean | `false` | +| disabled | When set to true, $[prodname] IPAM will not assign addresses from this pool. | true, false | boolean | `false` | +| disableBGPExport _(since v3.11.0)_ | Disable exporting routes from this IP Pool’s CIDR over BGP. | true, false | boolean | `false` | +| nodeSelector | Selects the nodes where $[prodname] IPAM should assign pod addresses from this pool. Can be overridden if a pod [explicitly identifies this IP pool by annotation](../component-resources/configuration.mdx#using-kubernetes-annotations). | | [selector](#node-selector) | all() | +| allowedUses _(since v3.11.0)_ | Controls whether the pool will be used for automatic assignments of certain types. See [below](#allowed-uses). | Workload, Tunnel, HostSecondaryInterface, LoadBalancer | list of strings | `["Workload", "Tunnel"]` | +| awsSubnetID _(since v3.11.0)_ | May be set to the ID of an AWS VPC Subnet that contains the CIDR of this IP pool to activate the AWS-backed pool feature. See [below](#aws-backed-pools). | Valid AWS Subnet ID. | string | | +| assignmentMode | Controls whether the pool will be used for automatic assignments or only if requested manually | Automatic, Manual | strings | `Automatic` | + +:::note + +Do not use a custom `blockSize` until **all** $[prodname] components have been updated to a version that +supports it (at least v2.3.0). Older versions of components do not understand the field so they may corrupt the +IP pool by creating blocks of incorrect size. + +::: + +### Allowed uses + +When automatically assigning IP addresses to workloads, only pools with "Workload" in their `allowedUses` field are +consulted. Similarly, when assigning IPs for tunnel devices, only "Tunnel" pools are eligible. Finally, when +assigning IP addresses for AWS secondary ENIs, only pools with allowed use "HostSecondaryInterface" are candidates. + +Combining options for the `allowedUses` field is limited. You can specify only the following options and option combinations: + * `allowedUses: ["Tunnel","Workload"]` (default) + * `allowedUses: ["Tunnel"]` + * `allowedUses: ["Workload"]` + * `allowedUses: ["LoadBalancer"]` + * `allowedUses: ["HostSecondaryInterface"]` + +If the `allowedUses` field is not specified, it defaults to `["Workload", "Tunnel"]` for compatibility with older +versions of Calico. It is not possible to specify a pool with no allowed uses. + +The `allowedUses` field is only consulted for new allocations, changing the field has no effect on previously allocated +addresses. + +$[prodname] supports Kubernetes [annotations that force the use of specific IP addresses](../component-resources/configuration.mdx#requesting-a-specific-ip-address). These annotations take precedence over the `allowedUses` field. + +### AWS-backed pools + +$[prodname] supports IP pools that are backed by the AWS fabric. This feature was added in order +to support egress gateways on the AWS fabric; the restrictions and requirements are currently documented as part of the +[egress gateways on AWS guide](../../networking/egress/egress-gateway-aws.mdx). + +### IPIP + +Routing of packets using IP-in-IP will be used when the destination IP address +is in an IP Pool that has IPIP enabled. In addition, if the `ipipMode` is set to `CrossSubnet`, +$[prodname] will only route using IP-in-IP if the IP address of the destination node is in a different +subnet. The subnet of each node is configured on the node resource (which may be automatically +determined when running the `$[nodecontainer]` service). + +For details on configuring IP-in-IP on your deployment, please refer to +[Configuring IP-in-IP](../../networking/configuring/vxlan-ipip.mdx). + +:::note + +Setting `natOutgoing` is recommended on any IP Pool with `ipip` enabled. +When `ipip` is enabled without `natOutgoing` routing between Workloads and +Hosts running $[prodname] is asymmetric and may cause traffic to be filtered due to +[RPF](https://en.wikipedia.org/wiki/Reverse_path_forwarding) checks failing. + +::: + +### VXLAN + +Routing of packets using VXLAN will be used when the destination IP address +is in an IP Pool that has VXLAN enabled. In addition, if the `vxlanMode` is set to `CrossSubnet`, +$[prodname] will only route using VXLAN if the IP address of the destination node is in a different +subnet. The subnet of each node is configured on the node resource (which may be automatically +determined when running the `$[nodecontainer]` service). + +:::note + +Setting `natOutgoing` is recommended on any IP Pool with `vxlan` enabled. +When `vxlan` is enabled without `natOutgoing` routing between Workloads and +Hosts running $[prodname] is asymmetric and may cause traffic to be filtered due to +[RPF](https://en.wikipedia.org/wiki/Reverse_path_forwarding) checks failing. + +::: + +### Block sizes + +The default block sizes of `26` for IPv4 and `122` for IPv6 provide blocks of 64 addresses. This allows addresses to be allocated in groups to workloads running on the same host. By grouping addresses, fewer routes need to be exchanged between hosts and to other BGP peers. If a host allocates all of the addresses in a block then it will be allocated an additional block. If there are no more blocks available then the host can take addresses from blocks allocated to other hosts. Specific routes are added for the borrowed addresses which has an impact on route table size. + +Increasing the block size from the default (e.g., using `24` for IPv4 to give 256 addresses per block) means fewer blocks per host, and potentially fewer routes. But try to ensure that there are at least as many blocks in the pool as there are hosts. + +Reducing the block size from the default (e.g., using `28` for IPv4 to give 16 addresses per block) means more blocks per host and therefore potentially more routes. This can be beneficial if it allows the blocks to be more fairly distributed amongst the hosts. + +### Node Selector + +For details on configuring IP pool node selectors, please read the +[Assign IP addresses based on topology guide.](../../networking/ipam/assign-ip-addresses-topology.mdx). + +#### Selector reference + + + +## Supported operations + +| Datastore type | Create/Delete | Update | Get/List | Notes | +| --------------------- | ------------- | ------ | -------- | ----- | +| Kubernetes API server | Yes | Yes | Yes | + +## See also + +The [`IPReservation` resource](ipreservation.mdx) allows for small parts of an IP pool to be reserved so that they will +not be used for automatic IPAM assignments. diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/ipreservation.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/ipreservation.mdx new file mode 100644 index 0000000000..b7e59b1bde --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/ipreservation.mdx @@ -0,0 +1,56 @@ +--- +description: Reference for the IPReservation resource in Calico Cloud connected clusters that excludes specific addresses or ranges from automatic allocation. +--- + +# IP reservation + +An IP reservation resource (`IPReservation`) represents a collection of IP addresses that $[prodname] should +not use when automatically assigning new IP addresses. It only applies when $[prodname] IPAM is in use. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: IPReservation +metadata: + name: my-ipreservation-1 +spec: + reservedCIDRs: + - 192.168.2.3 + - 10.0.2.3/32 + - cafe:f00d::/123 +``` + +## IP reservation definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | -------------------------------------------------- | --------------------------------------------------- | ------ | +| name | The name of this IPReservation resource. Required. | Alphanumeric string with optional `.`, `_`, or `-`. | string | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ------------- | --------------------------------------------------------------- | -------------------------------------------------- | ------ | ------- | +| reservedCIDRs | List of IP addresses and/or networks specified in CIDR notation | List of valid IP addresses (v4 or v6) and/or CIDRs | list | | + +### Notes + +The implementation of `IPReservation`s is designed to handle reservation of a small number of IP addresses/CIDRs from +(generally much larger) IP pools. If a significant portion of an IP pool is reserved (say more than 10%) then +$[prodname] may become significantly slower when searching for free IPAM blocks. + +Since `IPReservations` must be consulted for every IPAM assignment request, it's best to have one or two +`IPReservation` resources with multiple addresses per `IPReservation` resource (rather than having many IPReservation +resources), each with one address inside. + +If an `IPReservation` is created after an IP from its range is already in use then the IP is not automatically +released back to the pool. The reservation check is only done at auto allocation time. + +$[prodname] supports Kubernetes [annotations that force the use of specific IP addresses](../component-resources/configuration.mdx#requesting-a-specific-ip-address). These annotations override any `IPReservation`s that +are in place. + +When Windows nodes claim blocks of IPs they automatically assign the first three IPs +in each block and the final IP for internal purposes. These assignments cannot be blocked by an `IPReservation`. +However, if a whole IPAM block is reserved with an `IPReservation`, Windows nodes will not claim such a block. diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/kubecontrollersconfig.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/kubecontrollersconfig.mdx new file mode 100644 index 0000000000..ae90dc9d0f --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/kubecontrollersconfig.mdx @@ -0,0 +1,133 @@ +--- +description: Reference for the KubeControllersConfiguration resource in Calico Cloud connected clusters that controls behavior of the kube-controllers component. +--- + +import Selectors from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_selectors.mdx'; + +# Kubernetes controllers configuration + +A $[prodname] Kubernetes controllers configuration resource (`KubeControllersConfiguration`) configures the controllers that run inside the `$[imageNames.kubeControllers]` container. These controllers monitor the Kubernetes API and reconcile cluster state with the $[prodname] datastore. + +The `$[imageNames.kubeControllers]` container can run the following controllers; each is enabled by including a corresponding block under `spec.controllers`, and disabled by omitting the block. + +- **Node controller** — garbage collects IP addresses, cleans up $[prodname] node data when Kubernetes nodes are removed, and optionally creates and syncs host endpoints for each node. +- **Federated services controller** — watches Kubernetes services and endpoints locally and across all remote clusters defined through [RemoteClusterConfigurations](remoteclusterconfiguration.mdx), and programs Kubernetes endpoints for any locally-configured service that specifies a federation selector annotation. See [Configuring federated services](../../multicluster/services-controller.mdx) for the usage guide. +- **Load balancer controller** — manages IPAM for Services of type `LoadBalancer`. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: KubeControllersConfiguration +metadata: + name: default +spec: + logSeverityScreen: Info + healthChecks: Enabled + prometheusMetricsPort: 9094 + controllers: + node: + reconcilerPeriod: 5m + leakGracePeriod: 15m + syncLabels: Enabled + hostEndpoint: + autoCreate: Disabled + createDefaultHostEndpoint: Enabled + templates: + - generateName: custom-host-endpoint + interfaceCIDRs: + - 1.2.3.0/24 + interfacePattern: "eth0|eth1" + nodeSelector: "has(my-label)" + labels: + key: value + annotations: + key: value + loadbalancer: + assignIPs: AllServices +``` + +## Kubernetes controllers configuration definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | --------------------------------------------------------- | ----------------- | ------ | +| name | Unique name to describe this resource instance. Required. | Must be `default` | string | + +- $[prodname] automatically creates a resource named `default` containing the configuration settings, only the name `default` is used and only one object of this type is allowed. +{/*TODO-XREF-CC You can use [calicoctl](/reference/clis/calicoctl/overview/) to view and edit these settings.*/} + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| --------------------- | --------------------------------------------------------- | ----------------------------------- | --------------------------- | ------- | +| logSeverityScreen | The log severity above which logs are sent to the stdout. | Debug, Info, Warning, Error, Fatal | string | Info | +| healthChecks | Enable support for health checks | Enabled, Disabled | string | Enabled | +| prometheusMetricsPort | Port on which to serve prometheus metrics. | Set to 0 to disable, > 0 to enable. | TCP port | 9094 | +| controllers | Enabled controllers and their settings | | [Controllers](#controllers) | | + +### Controllers + +| Field | Description | Schema | +| ----------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------- | +| node | Enable and configure the node controller | omit to disable, or [NodeController](#nodecontroller) | +| federatedservices | Enable and configure the federated services controller | omit to disable, or [FederatedServicesController](#federatedservicescontroller) | + +### NodeController + +The node controller automatically cleans up configuration for nodes that no longer exist. Optionally, it can create host endpoints for all Kubernetes nodes. + +| Field | Description | Accepted Values | Schema | Default | +| ---------------- | -------------------------------------------------------------------------------- | ----------------- | --------------------------------- | ------- | +| reconcilerPeriod | Period to perform reconciliation with the $[prodname] datastore | | [Duration string][parse-duration] | 5m | +| syncLabels | When enabled, Kubernetes node labels will be copied to $[prodname] node objects. | Enabled, Disabled | string | Enabled | +| hostEndpoint | Configures the host endpoint controller | | [HostEndpoint](#hostendpoint) | | +| leakGracePeriod | Grace period to use when garbage collecting suspected leaked IP addresses. | | [Duration string][parse-duration] | 15m | + +### HostEndpoint + +| Field | Description | Accepted Values | Schema | Default | +| ------------------------- | ---------------------------------------------------------------- | ----------------- | --------------------- | -------- | +| autoCreate | When enabled, automatically create host endpoints | Enabled, Disabled | string | Disabled | +| createDefaultHostEndpoint | When enabled, default host endpoint will be created | Enabled, Disabled | string | Enabled | +| templates | Controls creation of custom host endpoints | | [Template](#template) | | + +### Template + +| Field | Description | Accepted Values | Schema | Default | +| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- | ------------------- | ----------------------------------- | -------- | +| generateName | Unique name used as suffix for host endpoints created based on this template | Alphanumeric string | string | | +| nodeSelector | Selects the nodes for which this template should create host endpoints | | [Selector](#selectors) | all() | +| interfaceCIDRs | This configuration defines which IP addresses from a node's specification (including standard, tunnel, and WireGuard IPs) are eligible for inclusion in the generated HostEndpoint. IP addresses must fall within the provided CIDR ranges to be considered. If no address on the node matches the specified CIDRs, the HostEndpoint creation is skipped. | List of valid CIDRs | List string | | +| interfacePattern | Regex to include matching interfaces and their IPs | string | string | | +| labels | Labels to be added to generated host endpoints matching this template | | map of string key to string values | | +| annotations | Annotations to be added to generated host endpoints matching this template | | map of string key to string values | | + +### Selectors + + + +### FederatedServicesController + +The federated services controller syncs Kubernetes services from remote clusters defined through [RemoteClusterConfigurations](remoteclusterconfiguration.mdx). It must have read access to the Kubernetes API to monitor `Service` and `Endpoints` events, and write access to update `Endpoints`. For the usage guide, see [Configuring federated services](../../multicluster/services-controller.mdx). + +| Field | Description | Schema | Default | +| ---------------- | ---------------------------------------------------------------- | --------------------------------- | ------- | +| reconcilerPeriod | Period to perform reconciliation with the $[prodname] datastore | [Duration string][parse-duration] | 5m | + +### LoadBalancerController + +The load balancer controller manages IPAM for Services of type LoadBalancer. + +| Field | Description | Accepted Values | Schema | Default | +| ---------------- | ---------------------------------------------------------------- | ---------------------------------------- | ------ | ----------- | +| assignIPs | Mode in which LoadBalancer controller operates | AllServices, RequestedServicesOnly | String | AllServices | + +## Supported operations + +| Datastore type | Create | Delete (Global `default`) | Update | Get/List | Notes | +| --------------------- | ------ | ------------------------- | ------ | -------- | ----- | +| Kubernetes API server | Yes | Yes | Yes | Yes | + +[parse-duration]: https://golang.org/pkg/time/#ParseDuration diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/licensekey.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/licensekey.mdx new file mode 100644 index 0000000000..8607d5df60 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/licensekey.mdx @@ -0,0 +1,82 @@ +--- +description: Reference for the LicenseKey resource in Calico Cloud connected clusters that activates entitled features. +--- + +# License key + +A License Key resource (`LicenseKey`) represents a user's license to use $[prodname]. Keys are +provided by Tigera support, and must be applied to the cluster to enable +$[prodname] features. + +For `kubectl` commands, the following case-insensitive aliases may be used to specify +the resource type on the CLI: `licensekey.projectcalico.org`, `licensekeys.projectcalico.org` +as well as abbreviations such as `licensekey.p` and `licensekeys.p`. + +## Working with license keys + +### Applying or updating a license key + +When you add $[prodname] to an existing Kubernetes cluster or create a +new OpenShift cluster, you must apply your license key to complete the installation +and gain access to the full set of $[prodname] features. + +When your license key expires, you must update it to continue using $[prodname]. + +To apply or update a license key use the following command, replacing `` +with the customer name in the file sent to you by Tigera. + +**Command** + +```bash +kubectl apply -f -license.yaml +``` + +**Example** + +```bash +kubectl apply -f awesome-corp-license.yaml +``` + +### Viewing information about your license key + +To view the number of licensed nodes and the license key expiry, use: + +```bash +kubectl get licensekeys.p -o custom-columns='Name:.metadata.name,MaxNodes:.status.maxnodes,Expiry:.status.expiry,PackageType:.status.package' +``` + +This is an example of the output of above command. + +``` +Name MaxNodes Expiry Package +default 100 2021-10-01T23:59:59Z Enterprise +``` + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: LicenseKey +metadata: + creationTimestamp: null + name: default +spec: + certificate: | + -----BEGIN CERTIFICATE----- + MII...n5 + -----END CERTIFICATE----- + token: eyJ...zaQ +status: + expiry: '2021-10-01T23:59:59Z' + maxnodes: 100 + package: Enterprise +``` + +The data fields in the license key resource may change without warning. The license key resource +is currently a singleton: the only valid name is `default`. + +## Supported operations + +| Datastore type | Create | Delete | Update | Get/List | Notes | +| --------------------- | ------ | ------ | ------ | -------- | ----- | +| Kubernetes API server | Yes | No | Yes | Yes | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/managedcluster.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/managedcluster.mdx new file mode 100644 index 0000000000..37e64371d1 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/managedcluster.mdx @@ -0,0 +1,70 @@ +--- +description: Reference for the ManagedCluster resource in Calico Cloud that registers a workload cluster with the Calico Cloud management plane. +--- + +# Managed Cluster + +A Managed Cluster resource (`ManagedCluster`) represents a cluster managed by a centralized management plane with a shared Elasticsearch. +The management plane provides central control of the managed cluster and stores its logs. + +$[prodname] supports connecting multiple $[prodname] clusters as describe in the [Multi-cluster management] installation guide. + +For `kubectl` commands, the following case-insensitive aliases may be used to specify the resource type on the CLI: +`managedcluster`,`managedclusters`, `managedcluster.projectcalico.org`, `managedclusters.projectcalico.org` as well as +abbreviations such as `managedcluster.p` and `managedclusters.p`. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: ManagedCluster +metadata: + name: managed-cluster +spec: + operatorNamespace: tigera-operator +``` + +## Managed cluster definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | --------------------------------------------------------- | --------------------------------------------------- | ------ | +| name | Unique name to describe this resource instance. Required. | Alphanumeric string with optional `.`, `_`, or `-`. | string | + +- `cluster` is a reserved name for the management plane and is considered an invalid value + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| -------------------- | ----------------------------------------------------------------------------------------------------------------- | --------------- | ------ | ------- | +| installationManifest | Installation Manifest to be applied on a managed cluster infrastructure | None | string | `Empty` | +| operatorNamespace | The namespace of the managed cluster's operator. This value is used in the generation of the InstallationManifest | None | string | `Empty` | + +- `installationManifest` field can be retrieved only once at creation time. Updates are not supported for this field. + +To extract the installation manifest at creation time `-o jsonpath="{.spec.installationManifest}"` parameters +can be used with a `kubectl` command. + +### Status + +Status represents the latest observed status of Managed cluster. The `status` is read-only for users and updated by the +$[prodname] components. + +| Field | Description | Schema | +| ---------- | -------------------------------------------------------------------------- | -------------------------------------- | +| conditions | List of condition that describe the current status of the Managed cluster. | List of ManagedClusterStatusConditions | + +**ManagedClusterStatusConditions** + +Conditions represent the latest observed set of conditions for a Managed cluster. The connection between a management +plane and managed plane will be reported as following: + +- `Unknown` when no initial connection has been established +- `True` when both planes have an established connection +- `False` when neither planes have an established connection + +| Field | Description | Accepted Values | Schema | Default | +| ------ | ------------------------------------------------------------------------- | -------------------------- | ------ | ------------------------- | +| type | Type of status that is being reported | - | string | `ManagedClusterConnected` | +| status | Status of the connection between a Managed cluster and management cluster | `Unknown`, `True`, `False` | string | `Unknown` | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/networkpolicy.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/networkpolicy.mdx new file mode 100644 index 0000000000..aa93970285 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/networkpolicy.mdx @@ -0,0 +1,155 @@ +--- +description: Reference for the NetworkPolicy resource in Calico Cloud, a namespaced tiered policy that selects pods within a single namespace in a connected cluster. +--- + +# Network policy + +import Servicematch from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_servicematch.mdx'; + +import Serviceaccountmatch from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_serviceaccountmatch.mdx'; + +import Ports from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_ports.mdx'; + +import SelectorScopes from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_selector-scopes.mdx'; + +import Selectors from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_selectors.mdx'; + +import Entityrule from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_entityrule.mdx'; + +import Icmp from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_icmp.mdx'; + +import Rule from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_rule.mdx'; + +A network policy resource (`NetworkPolicy`) represents an ordered set of rules which are applied +to a collection of endpoints that match a [label selector](#selector). + +`NetworkPolicy` is a namespaced resource. `NetworkPolicy` in a specific namespace +only applies to [workload endpoint resources](workloadendpoint.mdx) +in that namespace. Two resources are in the same namespace if the `namespace` +value is set the same on both. +See [global network policy resource](globalnetworkpolicy.mdx) for non-namespaced network policy. + +`NetworkPolicy` resources can be used to define network connectivity rules between groups of $[prodname] endpoints and host endpoints. + +{/*TODO-XREF-CC +and take precedence over [profile resources](/reference/resources/profile) if any are defined. +*/} + +NetworkPolicies are organized into [tiers](tier.mdx), which provide an additional layer of ordering—in particular, note that the `Pass` action skips to the +next [tier](tier.mdx), to enable hierarchical security policy. + +For `kubectl` [commands](https://kubernetes.io/docs/reference/kubectl/overview/), the following case-insensitive aliases +may be used to specify the resource type on the CLI: +`networkpolicy.projectcalico.org`, `networkpolicies.projectcalico.org` and abbreviations such as +`networkpolicy.p` and `networkpolicies.p`. + +## Sample YAML + +This sample policy allows TCP traffic from `frontend` endpoints to port 6379 on +`database` endpoints. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: internal-access.allow-tcp-6379 + namespace: production +spec: + tier: internal-access + selector: role == 'database' + types: + - Ingress + - Egress + ingress: + - action: Allow + metadata: + annotations: + from: frontend + to: database + protocol: TCP + source: + selector: role == 'frontend' + destination: + ports: + - 6379 + egress: + - action: Allow +``` + +## Definition + +### Metadata + +| Field | Description | Accepted Values | Schema | Default | +| --------- | ------------------------------------------------------------------ | --------------------------------------------------- | ------ | --------- | +| name | The name of the network policy. Required. | Alphanumeric string with optional `.`, `_`, or `-`. | string | | +| namespace | Namespace provides an additional qualification to a resource name. | | string | "default" | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------- | --------------------- | --------------------------------------------- | +| order | Controls the order of precedence. $[prodname] applies the policy with the lowest value first. | | float | | +| tier | Name of the [tier](tier.mdx) this policy belongs to. | | string | `default` | +| selector | Selects the endpoints to which this policy applies. | | [selector](#selector) | all() | +| types | Applies the policy based on the direction of the traffic. To apply the policy to inbound traffic, set to `Ingress`. To apply the policy to outbound traffic, set to `Egress`. To apply the policy to both, set to `Ingress, Egress`. | `Ingress`, `Egress` | List of strings | Depends on presence of ingress/egress rules\* | +| ingress | Ordered list of ingress rules applied by policy. | | List of [Rule](#rule) | | +| egress | Ordered list of egress rules applied by this policy. | | List of [Rule](#rule) | | +| serviceAccountSelector | Selects the service account(s) to which this policy applies. Select a specific service account by name using the `projectcalico.org/name` label. | | [selector](#selector) | all() | +| performanceHints | Contains a list of hints to Calico's policy engine to help process the policy more efficiently. Hints never change the enforcement behaviour of the policy. The available hints are described [below](#performance-hints). | `AssumeNeededOnEveryNode` | List of strings | | + +\* If `types` has no value, $[prodname] defaults as follows. + +> | Ingress Rules Present | Egress Rules Present | `Types` value | +> | --------------------- | -------------------- | ----------------- | +> | No | No | `Ingress` | +> | Yes | No | `Ingress` | +> | No | Yes | `Egress` | +> | Yes | Yes | `Ingress, Egress` | + +### Rule + + + +### ICMP + + + +### EntityRule + + + +### Selector + + + + +### Ports + + + +### ServiceAccountMatch + + + +### ServiceMatch + + + +### Performance Hints + +Performance hints provide a way to tell $[prodname] about the intended use of the policy so that it may +process it more efficiently. Currently only one hint is defined: + +* `AssumeNeededOnEveryNode`: normally, $[prodname] only calculates a policy's rules and selectors on nodes where + the policy is actually in use (i.e. its selector matches a local endpoint). This saves work in most cases. + The `AssumeNeededOnEveryNode` hint tells $[prodname] to treat the policy as "in use" on *every* node. This is + useful for large policy sets that are known to apply to all (or nearly all) endpoints. It effectively "preloads" + the policy on every node so that there is less work to do when the first endpoint matching the policy shows up. + It also prevents work from being done to tear down the policy when the last endpoint is drained. + +## Supported operations + +| Datastore type | Create/Delete | Update | Get/List | Notes | +| ------------------------ | ------------- | ------ | -------- | ----- | +| Kubernetes API datastore | Yes | Yes | Yes | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/networkset.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/networkset.mdx new file mode 100644 index 0000000000..17ee54c649 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/networkset.mdx @@ -0,0 +1,71 @@ +--- +description: Reference for the NetworkSet resource in Calico Cloud connected clusters that defines a namespaced set of CIDRs referenced by tiered network policy. +--- + +# Network set + +import DomainNames from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_domain-names.mdx'; + +A network set resource (`NetworkSet`) represents an arbitrary set of IP subnetworks/CIDRs, +allowing it to be matched by $[prodname] policy. Network sets are useful for applying policy to traffic +coming from (or going to) external, non-$[prodname], networks. + +`NetworkSet` is a namespaced resource. A `NetworkSet` in a specific namespace +only applies to [network policies](networkpolicy.mdx) +in that namespace. Two resources are in the same namespace if the `namespace` +value is set the same on both. (See [GlobalNetworkSet](globalnetworkset.mdx) for non-namespaced network sets.) + +The metadata for each network set includes a set of labels. When $[prodname] is calculating the set of +IPs that should match a source/destination selector within a +[network policy](networkpolicy.mdx) rule, it includes +the CIDRs from any network sets that match the selector. + +:::note + +Since $[prodname] matches packets based on their source/destination IP addresses, +$[prodname] rules may not behave as expected if there is NAT between the $[prodname]-enabled node and the +networks listed in a network set. For example, in Kubernetes, incoming traffic via a service IP is +typically SNATed by the kube-proxy before reaching the destination host so $[prodname]'s workload +policy will see the kube-proxy's host's IP as the source instead of the real source. + +::: + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkSet +metadata: + name: external-database + namespace: staging + labels: + role: db +spec: + nets: + - 198.51.100.0/28 + - 203.0.113.0/24 + allowedEgressDomains: + - db.com + - '*.db.com' +``` + +## Network set definition + +### Metadata + +| Field | Description | Accepted Values | Schema | Default | +| --------- | ------------------------------------------------------------------ | ------------------------------------------------- | ------ | --------- | +| name | The name of this network set. Required. | Lower-case alphanumeric with optional `_` or `-`. | string | | +| namespace | Namespace provides an additional qualification to a resource name. | | string | "default" | +| labels | A set of labels to apply to this endpoint. | | map | | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------------- | ------ | ------- | +| nets | The IP networks/CIDRs to include in the set. | Valid IPv4 or IPv6 CIDRs, for example "192.0.2.128/25" | list | | +| allowedEgressDomains | The list of domain names that belong to this set and are honored in egress allow rules only. Domain names specified here only work to allow egress traffic from the cluster to external destinations. They don't work to _deny_ traffic to destinations specified by domain name, or to allow ingress traffic from _sources_ specified by domain name. | List of [exact or wildcard domain names](#exact-and-wildcard-domain-names) | list | | + +### Exact and wildcard domain names + + diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/node.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/node.mdx new file mode 100644 index 0000000000..399f7e23e4 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/node.mdx @@ -0,0 +1,90 @@ +--- +description: Reference for the Node resource in Calico Cloud connected clusters that represents a host running the cnx-node agent. +--- + +# Node + +A node resource (`Node`) represents a node running $[prodname]. When adding a host +to a $[prodname] cluster, a node resource needs to be created which contains the +configuration for the `$[nodecontainer]` instance running on the host. + +When starting a `$[nodecontainer]` instance, the name supplied to the instance should +match the name configured in the Node resource. + +By default, starting a `$[nodecontainer]` instance will automatically create a node resource +using the `hostname` of the compute host. + +This resource is not supported in `kubectl`. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: Node +metadata: + name: node-hostname +spec: + bgp: + asNumber: 64512 + ipv4Address: 10.244.0.1/24 + ipv6Address: 2001:db8:85a3::8a2e:370:7334/120 + ipv4IPIPTunnelAddr: 192.168.0.1 +``` + +## Definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | -------------------------------- | --------------------------------------------------- | ------ | +| name | The name of this node. Required. | Alphanumeric string with optional `.`, `_`, or `-`. | string | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| -------------------- | --------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | ----------------------------- | ------- | +| bgp | BGP configuration for this node. Omit if using $[prodname] for policy only. | | [BGP](#bgp) | +| ipv4VXLANTunnelAddr | IPv4 address of the VXLAN tunnel. This is system configured and should not be updated manually. | | string | +| ipv6VXLANTunnelAddr | IPv6 address of the VXLAN tunnel. This is system configured and should not be updated manually. | | string | +| vxlanTunnelMACAddr | MAC address of the VXLAN tunnel. This is system configured and should not be updated manually. | | string | +| vxlanTunnelMACAddrV6 | MAC address of the IPv6 VXLAN tunnel. This is system configured and should not be updated manually. | | string | +| orchRefs | Correlates this node to a node in another orchestrator. | | list of [OrchRefs](#orchref) | +| wireguard | WireGuard configuration for this node. This is applicable only if WireGuard is enabled in [Felix Configuration](felixconfig.mdx). | | [WireGuard](#wireguard) | +| loadBalancer | Load balancer configuration for this node. This is applicable only if the eBPF dataplane is enabled in [Felix Configuration](felixconfig.mdx). | | [LoadBalancer](#loadbalancer) | + + +### OrchRef + +| Field | Description | Accepted Values | Schema | Default | +| ------------ | ------------------------------------------------ | --------------- | ------ | ------- | +| nodeName | Name of this node according to the orchestrator. | | string | +| orchestrator | Name of the orchestrator. | k8s | string | + +### BGP + +| Field | Description | Accepted Values | Schema | Default | +| ----------------------- | -------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | ------- | +| asNumber | The AS Number of your `$[nodecontainer]`. | Optional. If omitted the global value is used (see [example modifying Global BGP settings](../../networking/configuring/bgp.mdx) for details about modifying the `asNumber` setting). | integer | +| ipv4Address | The IPv4 address and subnet exported as the next-hop for the $[prodname] endpoints on the host | The IPv4 address must be specified if BGP is enabled. | string | +| ipv6Address | The IPv6 address and subnet exported as the next-hop for the $[prodname] endpoints on the host | Optional | string | +| ipv4IPIPTunnelAddr | IPv4 address of the IP-in-IP tunnel. This is system configured and should not be updated manually. | Optional IPv4 address | string | +| routeReflectorClusterID | Enables this node as a route reflector within the given cluster | Optional IPv4 address | string | + +### WireGuard + +| Field | Description | Accepted Values | Schema | Default | +| -------------------- | ----------------------------------------------------------------------------------------- | --------------- | ------ | ------- | +| interfaceIPv4Address | The IP address and subnet for the IPv4 WireGuard interface created by Felix on this node. | Optional | string | +| interfaceIPv6Address | The IP address and subnet for the IPv6 WireGuard interface created by Felix on this node. | Optional | string | + +### LoadBalancer + +| Field | Description | Accepted Values | Schema | Default | +| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | ------ | ------- | +| maintenance | Indicates if the node is under maintenance, for the purposes of external traffic load balancing. eBPF dataplane only. This corresponds to the 'lb.projectcalico.org/maintenance' annotation added to Kubernetes node resources. | Optional: `ExcludeLocalBackends` (K8s node annotation value: `exclude-local-backends`), `None` (K8s node annotation value: `none`) | string | + +## Supported operations + +| Datastore type | Create/Delete | Update | Get/List | Notes | +| --------------------- | ------------- | ------ | -------- | ------------------------------------------------------------------ | +| Kubernetes API server | No | Yes | Yes | `$[nodecontainer]` data is directly tied to the Kubernetes nodes. | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/overview.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/overview.mdx new file mode 100644 index 0000000000..df99702b44 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/overview.mdx @@ -0,0 +1,116 @@ +--- +description: Reference overview of the Calico Cloud API resources, including the manifest format and how kubectl manages them in connected clusters. +--- + +# Resource definitions + +This section describes the set of valid resource types that can be managed +through `calicoctl` or `kubectl`. + +While resources may be supplied in YAML or JSON format, this guide provides examples in YAML. + +## Overview of resource structure + +The calicoctl commands for resource management (create, apply, delete, replace, get) +all take resource manifests as input. + +Each manifest may contain a single resource +(e.g. a profile resource), or a list of multiple resources (e.g. a profile and two +hostEndpoint resources). + +The general structure of a single resource is as follows: + +```yaml +apiVersion: projectcalico.org/v3 +kind: +metadata: + # Identifying information + name: + ... +spec: + # Specification of the resource + ... +``` + +### Schema + +| Field | Description | Accepted Values | Schema | +| ---------- | --------------------------------------------------------------------------------------- | -------------------- | ------------------------ | +| apiVersion | Indicates the version of the API that the data corresponds to. | projectcalico.org/v3 | string | +| kind | Specifies the type of resource described by the YAML document. | | [kind](#supported-kinds) | +| metadata | Contains information used to uniquely identify the particular instance of the resource. | | map | +| spec | Contains the resource specification. | | map | + +### Supported kinds + +The following resources are supported: + +- [AlertException](alertexception.mdx) +- [BGPConfiguration](bgpconfig.mdx) +- [BGPPeer](bgppeer.mdx) +- [DeepPacketInspection](deeppacketinspection.mdx) +- [EgressGatewayPolicy](egressgatewaypolicy.mdx) +- [FelixConfiguration](felixconfig.mdx) +- [GlobalAlert](globalalert.mdx) +- [GlobalNetworkPolicy](globalnetworkpolicy.mdx) +- [GlobalNetworkSet](globalnetworkset.mdx) +- [GlobalThreatFeed](globalthreatfeed.mdx) +- [HostEndpoint](hostendpoint.mdx) +- [IPPool](ippool.mdx) +- [IPReservation](ipreservation.mdx) +- [KubeControllersConfiguration](kubecontrollersconfig.mdx) +- [LicenseKey](licensekey.mdx) +- [ManagedCluster](managedcluster.mdx) +- [NetworkPolicy](networkpolicy.mdx) +- [NetworkSet](networkset.mdx) +- [Node](node.mdx) +- [PacketCapture](packetcapture.mdx) + {/*TODO-XREF-CC [Profile](/reference/resources/profile)*/} +- [RemoteClusterConfiguration](remoteclusterconfiguration.mdx) +- [StagedGlobalNetworkPolicy](stagedglobalnetworkpolicy.mdx) +- [StagedKubernetesNetworkPolicy](stagedkubernetesnetworkpolicy.mdx) +- [StagedNetworkPolicy](stagednetworkpolicy.mdx) +- [Tier](tier.mdx) +- [WorkloadEndpoint](workloadendpoint.mdx) + +### Resource name requirements + +Every resource must have the `name` field specified. Name must be unique within a namespace. +Name required when creating resources, and cannot be updated. +A valid resource name can have alphanumeric characters with optional `.`, `_`, or `-`. of up to 128 characters total. + +### Multiple resources in a single file + +A file may contain multiple resource documents specified in a YAML list format. For example, the following is the contents of a file containing two `HostEndpoint` resources: + +```yaml +- apiVersion: projectcalico.org/v3 + kind: HostEndpoint + metadata: + name: endpoint1 + labels: + type: database + spec: + interface: eth0 + node: host1 + profiles: + - prof1 + - prof2 + expectedIPs: + - 1.2.3.4 + - '00:bb::aa' +- apiVersion: projectcalico.org/v3 + kind: HostEndpoint + metadata: + name: endpoint2 + labels: + type: frontend + spec: + interface: eth1 + node: host1 + profiles: + - prof1 + - prof2 + expectedIPs: + - 1.2.3.5 +``` diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/packetcapture.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/packetcapture.mdx new file mode 100644 index 0000000000..596ad0c879 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/packetcapture.mdx @@ -0,0 +1,148 @@ +--- +description: Reference for the PacketCapture resource in Calico Cloud connected clusters that captures pcap files from selected workloads for offline analysis. +--- + +# PacketCapture + +import Selectors from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_selectors.mdx'; + +A Packet Capture resource (`PacketCapture`) represents captured live traffic for debugging microservices and application +interaction inside a Kubernetes cluster. + +$[prodname] supports selecting one or multiple [WorkloadEndpoints resources](workloadendpoint.mdx) +as described in the [Packet Capture] guide. + +For `kubectl` [commands](https://kubernetes.io/docs/reference/kubectl/overview/), the following case-insensitive aliases may be used to specify the resource type on the CLI: +`packetcapture`,`packetcaptures`, `packetcapture.projectcalico.org`, `packetcaptures.projectcalico.org` as well as +abbreviations such as `packetcapture.p` and `packetcaptures.p`. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: PacketCapture +metadata: + name: sample-capture + namespace: sample-namespace +spec: + selector: k8s-app == "sample-app" + filters: + - protocol: TCP + ports: + - 80 +``` + +```yaml +apiVersion: projectcalico.org/v3 +kind: PacketCapture +metadata: + name: sample-capture + namespace: sample-namespace +spec: + selector: all() + startTime: '2021-08-26T12:00:00Z' + endTime: '2021-08-26T12:30:00Z' +``` + +## Packet capture definition + +### Metadata + +| Field | Description | Accepted Values | Schema | Default | +| --------- | ------------------------------------------------------------------ | --------------------------------------------------- | ------ | --------- | +| name | The name of the packet capture. Required. | Alphanumeric string with optional `.`, `_`, or `-`. | string | | +| namespace | Namespace provides an additional qualification to a resource name. | | string | "default" | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| --------- | ---------------------------------------------------------------------------------- | ----------------------- | ----------------------- | ------- | +| selector | Selects the endpoints to which this packet capture applies. | | [selector](#selector) | | +| filters | The ordered set of filters applied to traffic captured from an interface. | | [filters](#filters) | | +| startTime | Defines the start time from which this PacketCapture will start capturing packets. | Date in RFC 3339 format | [startTime](#starttime) | | +| endTime | Defines the end time at which this PacketCapture will stop capturing packets. | Date in RFC 3339 format | [endTime](#endtime) | | + +### Selector + + + +### Filters + +| Field | Description | Accepted Values | Schema | Default | +| -------- | ------------------------------------- | ------------------------------------------------------------ | ----------------- | ------- | +| protocol | Positive protocol match. | `TCP`, `UDP`, `ICMP`, `ICMPv6`, `SCTP`, `UDPLite`, `1`-`255` | string \| integer | | +| ports | Positive match on the specified ports | | list of ports | | + +$[prodname] supports the following syntax for expressing ports. + +| Syntax | Example | Description | +| --------- | --------- | ------------------------------------------------------ | +| int | 80 | The exact (numeric) port specified | +| start:end | 6040:6050 | All (numeric) ports within the range start ≤ x ≤ end | + +An individual numeric port may be specified as a YAML/JSON integer. A port range must be represented as a string. Named ports are not supported by `PacketCapture`. +Multiple ports can be defined to filter traffic. All specified ports or port ranges concatenated using the logical operator "OR". + +For example, this would be a valid list of ports: + +```yaml +ports: [8080, '1234:5678'] +``` + +Multiple filter rules can be defined to filter traffic. All rules are concatenated using the logical operator "OR". +For example, filtering both TCP or UDP traffic will be defined as: + +```yaml +filters: + - protocol: TCP + - protocol: UDP +``` + +Within a single filter rule, protocol and list of valid ports will be concatenated using the logical operator "AND". + +For example, filtering TCP traffic and traffic for port 80 will be defined as: + +```yaml +filters: + - protocol: TCP + ports: [80] +``` + +### StartTime + +Defines the start time from which this PacketCapture will start capturing packets in RFC 3339 format. +If omitted or the value is in the past, the capture will start immediately. +If the value is changed to a future time, capture will stop immediately and restart at that time. + +```yaml +startTime: '2021-08-26T12:00:00Z' +``` + +### EndTime + +Defines the end time from which this PacketCapture will stop capturing packets in RFC 3339 format. +If omitted the capture will continue indefinitely. +If the value is changed to the past, capture will stop immediately. + +```yaml +endTime: '2021-08-26T12:30:00Z' +``` + +### Status + +`PacketCaptureStatus` lists the current state of a `PacketCapture` and its generated capture files. + +| Field | Description | +| ----- | -------------------------------------------------------------------------------------------------------------------------------- | +| files | It describes the location of the packet capture files that is identified via a node, its directory and the file names generated. | + +### Files + +| Field | Description | +| --------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| directory | The path inside the calico-node container for the generated files. | +| fileNames | The name of the generated file for a `PacketCapture` ordered alphanumerically.
The active packet capture file will be identified using the following schema: `{}.pcap`.
Rotated capture files name will contain an index matching the rotation timestamp. | +| node | The hostname of the Kubernetes node the files are located on. | +| state | Determines whether a PacketCapture is capturing traffic from any interface attached to the current node. Possible values include: Capturing, Scheduled, Finished, Error, WaitingForTraffic | + +[packet capture]: /observability/packetcapture.mdx \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/policyrecommendations.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/policyrecommendations.mdx new file mode 100644 index 0000000000..4a2437fc1a --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/policyrecommendations.mdx @@ -0,0 +1,56 @@ +--- +description: Reference for the PolicyRecommendation resource in Calico Cloud connected clusters that drives policy recommendations from observed namespace flows. +--- + +# Policy recommendation scope + +import Servicematch from '../../_includes/content/_servicematch.mdx'; + +import Serviceaccountmatch from '../../_includes/content/_serviceaccountmatch.mdx'; + +import Ports from '../../_includes/content/_ports.mdx'; + +import Selectors from '../../_includes/content/_selectors.mdx'; + +import Entityrule from '../../_includes/content/_entityrule.mdx'; + +import Icmp from '../../_includes/content/_icmp.mdx'; + +import Rule from '../../_includes/content/_rule.mdx'; + +The policy recommendation scope is a collection of configuration options to control [policy recommendation](../../network-policy/recommendations/policy-recommendations.mdx) in the web console. + +To apply changes to this resource, use the following format: + +``` +$ kubectl patch policyrecommendationscope default -p '{"spec":{"":""}}' +``` +**Example** + +`$ kubectl patch policyrecommendationscope default -p '{"spec":{"interval":"5m"}}'` + +## Definition + +### + +### Metadata + +| Field | Description | Accepted Values | Schema | Default | +| ---------------- | ------------------------------------------------------------ | --------------------------------------------------- | ------ | -------------------------------------------------- | +| name | The name of the policy recommendation scope. | `default` | string | | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ---------------------- | ------------------------------------------------------------ | --------------- | ------ | -------------- | +| Interval | The frequency to create and refine policy recommendations. | | | 2.5m (minutes) | +| InitialLookback | Start time to look at flow logs when first creating a policy recommendation. | | | 24h (hours) | +| StabilizationPeriod | Time that a recommended policy should remain unchanged so it is stable and ready to be enforced. | | | 10m (minutes) | + +#### NamespaceSpec + +| Field | Description | Accepted Values | Schema | Default | +| ---------------------- | ------------------------------------------------------------ | --------------- | ------ | -------------- | +| recStatus | Defines the policy recommendation engine status. | Enabled/Disabled | | Disabled | +| selector | Selects the namespaces for generating recommendations. | | | `!(projectcalico.org/name starts with ''tigera-'') && !(projectcalico.org/name starts with ''calico-'') && !(projectcalico.org/name starts with ''kube-'')` | +| intraNamespacePassThroughTraffic | When true, sets all intra-namespace traffic to Pass | true/false | | false | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/remoteclusterconfiguration.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/remoteclusterconfiguration.mdx new file mode 100644 index 0000000000..4ba8d77f7a --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/remoteclusterconfiguration.mdx @@ -0,0 +1,97 @@ +--- +description: Reference for the RemoteClusterConfiguration resource in Calico Cloud that federates resources between connected clusters for shared identity. +--- + +# Remote cluster configuration + +A remote cluster configuration resource (RemoteClusterConfiguration) represents a cluster in a federation of clusters. +Each remote cluster needs a configuration to be specified to allow the local cluster to access resources on the remote +cluster. The connection is one-way: the information flows only from the remote to the local cluster. To share +information from the local cluster to the remote one a remote cluster configuration resource must be created on the +remote cluster. + +A remote cluster configuration causes Typha and `calicoq` to retrieve the following resources from a remote cluster: + +- [Workload endpoints](workloadendpoint.mdx) +- [Host endpoints](hostendpoint.mdx) +{/*TODO-XREF-CC- [Profiles](/reference/resources/profile) (rules are not retrieved from remote profiles, only the `LabelsToApply` field is used)*/} + +When using the Kubernetes API datastore with RBAC enabled on the remote cluster, the RBAC rules must be configured to +allow access to these resources. + +For more details on the federation feature refer to the [Overview](../../multicluster/overview.mdx). + +{/*TODO-XREF-CC For the meaning of the fields matches the configuration used for configuring `calicoctl`, see [Kubernetes datastore](/operations/clis/calicoctl/configure/datastore) instructions for more details.*/} + +This resource is not supported in `kubectl`. + +## Sample YAML + +For a remote Kubernetes datastore cluster: + +```yaml +apiVersion: projectcalico.org/v3 +kind: RemoteClusterConfiguration +metadata: + name: cluster1 +spec: + datastoreType: kubernetes + kubeconfig: /etc/tigera-federation-remotecluster/kubeconfig-rem-cluster-1 +``` + +For a remote etcdv3 cluster: + +```yaml +apiVersion: projectcalico.org/v3 +kind: RemoteClusterConfiguration +metadata: + name: cluster1 +spec: + datastoreType: etcdv3 + etcdEndpoints: 'https://10.0.0.1:2379,https://10.0.0.2:2379' +``` + +## RemoteClusterConfiguration Definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | ---------------------------------------------- | ----------------------------------------- | ------ | +| name | The name of this remote cluster configuration. | Lower-case alphanumeric with optional `-` | string | + +### Spec + +| Field | Secret key | Description | Accepted Values | Schema | Default | +| ------------------- | ------------- | ------------------------------------------------------------------ | --------------------- | -------------------------- | ------- | +| clusterAccessSecret | | Reference to a Secret that contains connection information | | Kubernetes ObjectReference | none | +| datastoreType | datastoreType | The datastore type of the remote cluster. | `etcdv3` `kubernetes` | string | none | +| etcdEndpoints | etcdEndpoints | A comma separated list of etcd endpoints. | | string | none | +| etcdUsername | etcdUsername | Username for RBAC. | | string | none | +| etcdPassword | etcdPassword | Password for the given username. | | string | none | +| etcdKeyFile | etcdKey | Path to the etcd key file. | | string | none | +| etcdCertFile | etcdCert | Path to the etcd certificate file. | | string | none | +| etcdCACertFile | etcdCACert | Path to the etcd CA certificate file. | | string | none | +| kubeconfig | kubeconfig | Location of the `kubeconfig` file. | | string | none | +| k8sAPIEndpoint | | Location of the kubernetes API server. | | string | none | +| k8sKeyFile | | Location of a client key for accessing the Kubernetes API. | | string | none | +| k8sCertFile | | Location of a client certificate for accessing the Kubernetes API. | | string | none | +| k8sCAFile | | Location of a CA certificate. | | string | none | +| k8sAPIToken | | Token to be used for accessing the Kubernetes API. | | string | none | + +When using the `clusterAccessSecret` field, all other fields in the RemoteClusterconfiguration resource must be empty. +When the `clusterAccessSecret` reference is used, all datastore configuration will be read from the referenced Secret +using the "Secret key" fields named in the above table as the data key in the Secret. The fields read from a Secret +that were file path or locations in a RemoteClusterConfiguration will be expected to be the file contents when read +from a Secret. + +All of the fields that start with `etcd` are only valid when the DatastoreType is etcdv3 and the fields that start with `k8s` or `kube` are only valid when the datastore type is kubernetes. +The `kubeconfig` field and the fields that end with `File` must be accessible to Typha and `calicoq`, this does not apply when the data is coming from a Secret referenced by `clusterAccessSecret`. + +When the DatastoreType is `kubernetes`, the `kubeconfig` file is optional but since it can contain all of the authentication information needed to access the Kubernetes API server it is generally easier to use than setting all the individual `k8s` fields. The other kubernetes fields can be used by themselves though or to override specific kubeconfig values. + +## Supported operations + +| Datastore type | Create/Delete | Update | Get/List | Notes | +| --------------------- | ------------- | ------ | -------- | ----- | +| etcdv3 | Yes | Yes | Yes | +| Kubernetes API server | Yes | Yes | Yes | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/securityeventwebhook.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/securityeventwebhook.mdx new file mode 100644 index 0000000000..69baa6d59b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/securityeventwebhook.mdx @@ -0,0 +1,158 @@ +--- +description: Reference for the SecurityEventWebhook resource in Calico Cloud connected clusters that forwards security events to external systems such as Slack or Jira. +--- + +# Security event webhook + +A security event webhook (`SecurityEventWebhook`) is a cluster-scoped resource that represents instances +of integrations with external systems through the webhook callback mechanism. + +For `kubectl` [commands](https://kubernetes.io/docs/reference/kubectl/overview/), the following case-insensitive aliases +can be used to specify the resource type on the CLI: +`securityeventwebhook.projectcalico.org`, `securityeventwebhooks.projectcalico.org` and abbreviations such as +`securityeventwebhook.p` and `securityeventwebhooks.p`. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: SecurityEventWebhook +metadata: + name: jira-webhook + annotations: + webhooks.projectcalico.org/labels: 'Cluster name:Calico Enterprise' +spec: + consumer: Jira + state: Enabled + query: type=waf + config: + - name: url + value: 'https://your-jira-instance-name.atlassian.net/rest/api/2/issue/' + - name: project + value: PRJ + - name: issueType + value: Bug + - name: username + valueFrom: + secretKeyRef: + name: jira-secrets + key: username + - name: apiToken + valueFrom: + secretKeyRef: + name: jira-secrets + key: token +``` + +## Security event webhook definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | --------------------------------------------------------- | --------------------------------------------------- | ------ | +| name | Unique name to describe this resource instance. Required. | Alphanumeric string with optional `.`, `_`, or `-`. | string | + +#### Annotations + +Security event webhooks provide an easy way to add arbitrary data to the webhook generated HTTP payload through the metadata annotation. +The value of the `webhooks.projectcalico.org/labels`, if present, will be converted into the payload labels. +The value must conform to the following rules: + +- Key and value data for a single label are separated by the `:` character, +- Multiple labels are separated by the `,` character. + +### Spec + +| Field | Description | Accepted Values | Schema | Required | +| -------- | --------------------------------------------------------------------------------------------------------------- | ----------------------------------- | ------------------------------------------------------------------------| --------------------------- | +| consumer | Specifies intended consumer of the webhook. | Slack, Jira, Alertmanager, Generic | string | yes | +| state | Defines current state of the webhook. | Enabled, Disabled, Debug | string | yes | +| query | Defines query used to retrieve security events from Calico. | [see Query](#query) | string | yes | +| config | Webhook configuration, required contents of this structure is determined by the value of the `consumer` field. | [see Config](#configuration) | list of [SecurityEventWebhookConfigVar](#securityeventwebhookconfigvar) | yes | + +### SecurityEventWebhookConfigVar + +| Field | Description | Schema | Required | +| ------------ | -------------------------------------------------------------------------- | --------------------------------------------------------------------------- | ----------------------------------- | +| name | Configuration variable name. | string | yes | +| value | Direct value for the variable. | string | yes if `valueFrom` is not specified | +| valueFrom | Value defined either in a Kubernetes ConfigMap or in a Kubernetes Secret. | [SecurityEventWebhookConfigVarSource](#securityeventwebhookconfigvarsource) | yes if `value` is not specified | + +### SecurityEventWebhookConfigVarSource + +| Field | Description | Schema | Required | +| ---------------- | --------------------------------- | ------------------------------------------------------------------------------------------------------------ | ----------------------------------------- | +| configMapKeyRef | Kubernetes ConfigMap reference. | `ConfigMapKeySelector` (referenced ConfigMap key should exist in the `tigera-intrusion-detection` namespace) | yes if `secretKeyRef` is not specified | +| secretKeyRef | Kubernetes Secret reference. | `SecretKeySelector` (referenced Secret key should exist in the `tigera-intrusion-detection` namespace) | yes if `configMapKeyRef` is not specified | + +### Status + +Field `status` reflects the health of a webhook. It is a list of [Kubernetes Conditions](https://pkg.go.dev/k8s.io/apimachinery@v0.23.0/pkg/apis/meta/v1#Condition). + +## Query + +Security event webhooks use a domain-specific query language to select which records +from the data set should trigger the HTTP request. + +The query language is composed of any number of selectors, combined +with boolean expressions (`AND`, `OR`, and `NOT`), set expressions +(`IN` and `NOTIN`) and bracketed subexpressions. These are translated +by $[prodname] to Elastic DSL queries that are executed on the backend. + +Set expressions support wildcard operators asterisk (`*`) and question mark (`?`). +The asterisk sign matches zero or more characters and the question mark matches a single character. + +A selector consists of a key, comparator, and value. Keys and values +may be identifiers consisting of alphanumerics and underscores (`_`) +with the first character being alphabetic or an underscore, or may be +quoted strings. Values may also be integer or floating point numbers. +Comparators may be `=` (equal), `!=` (not equal), `<` (less than), +`<=` (less than or equal), `>` (greater than), or `>=` (greater than +or equal). + +## Configuration + +Data required to be present in the `config` section of the security event webhook `spec` depends on the intended consumer for the HTTP +requests generated by the webhook. The value in the `consumer` field of the `spec` specifies the consumer and therefore data +that is required to be present. Currently Calico supports the following consumers: `Slack`, `Jira`, `Alertmanager` and `Generic`. +Payloads generated by the webhook will be different for each of the listed use cases. + +### Slack + +Data fields required for the `Slack` value present in the `spec.consumer` field of a webhook: + +| Field | Description | Required | +| ---------------- | ------------------------------------------------------------------------------ | ---------- | +| url | A valid Slack [Incoming Webhook URL](https://api.slack.com/messaging/webhooks). | yes | + +### Generic + +Data fields required for the `Generic` value present in the `spec.consumer` field of a webhook: + +| Field | Description | Required | +| ---------------- | --------------------------------------------------- | ---------- | +| url | A generic and valid URL of another HTTP(s) endpoint. | yes | + +### Jira + +Data fields required for the `Jira` value present in the `spec.consumer` field of a webhook: + +| Field | Description | Required | +| ---------------- | ---------------------------------------------------------------------- | ---------- | +| url | URL of a Jira REST API v2 endpoint for the organisation. | yes | +| project | A valid Jira project abbreviation. | yes | +| issueType | A valid issue type for the selected project, examples: `Bug` or `Task` | yes | +| username | A valid Jira user name. | yes | +| apiToken | A valid Jira API token for the user. | yes | + +### Alertmanager + +Data fields required for the `Alertmanager` value present in the `spec.consumer` field of a webhook: + +| Field | Description | Required | +| ---------------- | --------------------------------------------------------------------------------------- | ---------- | +| url | URL of the Alertmanager REST API v2 endpoint for alerts (ending with `/api/v2/alerts`). | yes | +| basicAuth | MD5 checksum of username and password separated by the colon character. | no | +| ca.crt | Certificate authority in PEM format (required for mTLS configuration). | no | +| tls.key | Private key in PEM format (required for mTLS configuration). | no | +| tls.crt | Certificate in PEM format (required for mTLS configuration). | no | \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/stagedglobalnetworkpolicy.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/stagedglobalnetworkpolicy.mdx new file mode 100644 index 0000000000..c34b1e18bd --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/stagedglobalnetworkpolicy.mdx @@ -0,0 +1,164 @@ +--- +description: Reference for the StagedGlobalNetworkPolicy resource in Calico Cloud connected clusters that previews cluster-scoped tiered policy without enforcing it. +--- + +# Staged global network policy + +import Servicematch from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_servicematch.mdx'; + +import Serviceaccountmatch from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_serviceaccountmatch.mdx'; + +import Ports from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_ports.mdx'; + +import Selectors from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_selectors.mdx'; + +import Entityrule from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_entityrule.mdx'; + +import Icmp from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_icmp.mdx'; + +import Rule from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_rule.mdx'; + +A staged global network policy resource (`StagedGlobalNetworkPolicy`) represents an ordered set of rules which are applied +to a collection of endpoints that match a [label selector](#selector). These rules are used to preview network behavior and do +not enforce network traffic. For enforcing network traffic, see [global network policy resource](globalnetworkpolicy.mdx). + +`StagedGlobalNetworkPolicy` is not a namespaced resource. `StagedGlobalNetworkPolicy` applies to [workload endpoint resources](workloadendpoint.mdx) in all namespaces, and to [host endpoint resources](hostendpoint.mdx). +Select a namespace in a `StagedGlobalNetworkPolicy` in the standard selector by using +`projectcalico.org/namespace` as the label name and a `namespace` name as the +value to compare against, e.g., `projectcalico.org/namespace == "default"`. +See [staged network policy resource](stagednetworkpolicy.mdx) for staged namespaced network policy. + +`StagedGlobalNetworkPolicy` resources can be used to define network connectivity rules between groups of $[prodname] endpoints and host endpoints. +{/*TODO-XREF-CC +and take precedence over [Profile resources](/reference/resources/profile) if any are defined. +*/} + +StagedGlobalNetworkPolicies are organized into [tiers](tier.mdx), which provide an additional layer of ordering—in particular, note that the `Pass` action skips to the +next [tier](tier.mdx), to enable hierarchical security policy. + +For `kubectl` [commands](https://kubernetes.io/docs/reference/kubectl/overview/), the following case-insensitive aliases +may be used to specify the resource type on the CLI: +`stagedglobalnetworkpolicy.projectcalico.org`, `stagedglobalnetworkpolicies.projectcalico.org` and abbreviations such as +`stagedglobalnetworkpolicy.p` and `stagedglobalnetworkpolicies.p`. + +## Sample YAML + +This sample policy allows TCP traffic from `frontend` endpoints to port 6379 on +`database` endpoints. + +```yaml +apiVersion: projectcalico.org/v3 +kind: StagedGlobalNetworkPolicy +metadata: + name: internal-access.allow-tcp-6379 +spec: + tier: internal-access + selector: role == 'database' + types: + - Ingress + - Egress + ingress: + - action: Allow + protocol: TCP + source: + selector: role == 'frontend' + destination: + ports: + - 6379 + egress: + - action: Allow +``` + +## Definition + +### Metadata + +| Field | Description | Accepted Values | Schema | Default | +| ----- | ----------------------------------------- | --------------------------------------------------- | ------ | ------- | +| name | The name of the network policy. Required. | Alphanumeric string with optional `.`, `_`, or `-`. | string | | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------- | --------------------- | --------------------------------------------- | +| order | Controls the order of precedence. $[prodname] applies the policy with the lowest value first. | | float | | +| tier | Name of the [tier](tier.mdx) this policy belongs to. | | string | `default` | +| selector | Selects the endpoints to which this policy applies. | | [selector](#selector) | all() | +| serviceAccountSelector | Selects the service account(s) to which this policy applies. Select all service accounts in the cluster with a specific name using the `projectcalico.org/name` label. | | [selector](#selector) | all() | +| namespaceSelector | Selects the namespace(s) to which this policy applies. Select a specific namespace by name using the `projectcalico.org/name` label. | | [selector](#selector) | all() | +| types | Applies the policy based on the direction of the traffic. To apply the policy to inbound traffic, set to `Ingress`. To apply the policy to outbound traffic, set to `Egress`. To apply the policy to both, set to `Ingress, Egress`. | `Ingress`, `Egress` | List of strings | Depends on presence of ingress/egress rules\* | +| ingress | Ordered list of ingress rules applied by policy. | | List of [Rule](#rule) | | +| egress | Ordered list of egress rules applied by this policy. | | List of [Rule](#rule) | | +| doNotTrack\*\* | Indicates to apply the rules in this policy before any data plane connection tracking, and that packets allowed by these rules should not be tracked. | true, false | boolean | false | +| preDNAT\*\* | Indicates to apply the rules in this policy before any DNAT. | true, false | boolean | false | +| applyOnForward\*\* | Indicates to apply the rules in this policy on forwarded traffic as well as to locally terminated traffic. | true, false | boolean | false | +| performanceHints | Contains a list of hints to Calico's policy engine to help process the policy more efficiently. Hints never change the enforcement behaviour of the policy. The available hints are described [below](#performance-hints). | `AssumeNeededOnEveryNode` | List of strings | | + +\* If `types` has no value, $[prodname] defaults as follows. + +> | Ingress Rules Present | Egress Rules Present | `Types` value | +> | --------------------- | -------------------- | ----------------- | +> | No | No | `Ingress` | +> | Yes | No | `Ingress` | +> | No | Yes | `Egress` | +> | Yes | Yes | `Ingress, Egress` | + +\*\* The `doNotTrack` and `preDNAT` and `applyOnForward` fields are meaningful +only when applying policy to a [host endpoint](hostendpoint.mdx). + +Only one of `doNotTrack` and `preDNAT` may be set to `true` (in a given policy). If they are both `false`, or when applying the policy to a +[workload endpoint](workloadendpoint.mdx), +the policy is enforced after connection tracking and any DNAT. + +`applyOnForward` must be set to `true` if either `doNotTrack` or `preDNAT` is +`true` because for a given policy, any untracked rules or rules before DNAT will +in practice apply to forwarded traffic. + +See [Using $[prodname] to Secure Host Interfaces](../host-endpoints/index.mdx) +for how `doNotTrack` and `preDNAT` and `applyOnForward` can be useful for host endpoints. + +### Rule + + + +### ICMP + + + +### EntityRule + + + +### Selector + + + +### Ports + + + +### ServiceAccountMatch + + + +### ServiceMatch + + + +### Performance Hints + +Performance hints provide a way to tell $[prodname] about the intended use of the policy so that it may +process it more efficiently. Currently only one hint is defined: + +* `AssumeNeededOnEveryNode`: normally, $[prodname] only calculates a policy's rules and selectors on nodes where + the policy is actually in use (i.e. its selector matches a local endpoint). This saves work in most cases. + The `AssumeNeededOnEveryNode` hint tells $[prodname] to treat the policy as "in use" on *every* node. This is + useful for large policy sets that are known to apply to all (or nearly all) endpoints. It effectively "preloads" + the policy on every node so that there is less work to do when the first endpoint matching the policy shows up. + It also prevents work from being done to tear down the policy when the last endpoint is drained. + +## Supported operations + +| Datastore type | Create/Delete | Update | Get/List | Notes | +| ------------------------ | ------------- | ------ | -------- | ----- | +| Kubernetes API datastore | Yes | Yes | Yes | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/stagedkubernetesnetworkpolicy.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/stagedkubernetesnetworkpolicy.mdx new file mode 100644 index 0000000000..22c30d274b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/stagedkubernetesnetworkpolicy.mdx @@ -0,0 +1,65 @@ +--- +description: Reference for the StagedKubernetesNetworkPolicy resource in Calico Cloud connected clusters that previews Kubernetes network policy without enforcing it. +--- + +# Staged Kubernetes network policy + +A staged Kubernetes network policy resource (`StagedKubernetesNetworkPolicy`) represents a staged version +of [Kubernetes network policy](https://kubernetes.io/docs/concepts/services-networking/network-policies). +This is used to preview network behavior before actually enforcing the network policy. Once persisted, this +will create a Kubernetes network policy backed by a $[prodname] +[network policy](networkpolicy.mdx). + +For `kubectl` [commands](https://kubernetes.io/docs/reference/kubectl/overview/), the following case-insensitive aliases +may be used to specify the resource type on the CLI: +`stagedkubernetesnetworkpolicy.projectcalico.org`, `stagedkubernetesnetworkpolicies.projectcalico.org` and abbreviations such as +`stagedkubernetesnetworkpolicy.p` and `stagedkubernetesnetworkpolicies.p`. + +## Sample YAML + +Below is a sample policy created from the example policy from the +[Kubernetes NetworkPolicy documentation](https://kubernetes.io/docs/concepts/services-networking/network-policies/#networkpolicy-resource). +The only difference between this policy and the example Kubernetes version is that the `apiVersion` and `kind` are changed +to properly specify a staged Kubernetes network policy. + +```yaml +apiVersion: projectcalico.org/v3 +kind: StagedKubernetesNetworkPolicy +metadata: + name: test-network-policy + namespace: default +spec: + podSelector: + matchLabels: + role: db + policyTypes: + - Ingress + - Egress + ingress: + - from: + - ipBlock: + cidr: 172.17.0.0/16 + except: + - 172.17.1.0/24 + - namespaceSelector: + matchLabels: + project: myproject + - podSelector: + matchLabels: + role: frontend + ports: + - protocol: TCP + port: 6379 + egress: + - to: + - ipBlock: + cidr: 10.0.0.0/24 + ports: + - protocol: TCP + port: 5978 +``` + +## Definition + +See the [Kubernetes NetworkPolicy documentation](https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#networkpolicyspec-v1-networking-k8s-io) +for more information. diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/stagednetworkpolicy.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/stagednetworkpolicy.mdx new file mode 100644 index 0000000000..adf654a6da --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/stagednetworkpolicy.mdx @@ -0,0 +1,148 @@ +--- +description: Reference for the StagedNetworkPolicy resource in Calico Cloud connected clusters that previews namespaced tiered policy without enforcing it. +--- + +# Staged network policy + +import Servicematch from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_servicematch.mdx'; + +import Serviceaccountmatch from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_serviceaccountmatch.mdx'; + +import Ports from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_ports.mdx'; + +import Selectors from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_selectors.mdx'; + +import Entityrule from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_entityrule.mdx'; + +import Icmp from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_icmp.mdx'; + +import Rule from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_rule.mdx'; + +A staged network policy resource (`StagedNetworkPolicy`) represents an ordered set of rules which are applied +to a collection of endpoints that match a [label selector](#selector). These rules are used to preview network behavior and do +not enforce network traffic. For enforcing network traffic, see [network policy resource](networkpolicy.mdx). + +`StagedNetworkPolicy` is a namespaced resource. `StagedNetworkPolicy` in a specific namespace +only applies to [workload endpoint resources](workloadendpoint.mdx) +in that namespace. Two resources are in the same namespace if the `namespace` +value is set the same on both. +See [staged global network policy resource](stagedglobalnetworkpolicy.mdx) for staged non-namespaced network policy. + +`StagedNetworkPolicy` resources can be used to define network connectivity rules between groups of $[prodname] endpoints and host endpoints. +{/*TODO-XREF-CC +and take precedence over [profile resources](/reference/resources/profile) if any are defined. +*/} + +StagedNetworkPolicies are organized into [tiers](tier.mdx), which provide an additional layer of ordering—in particular, note that the `Pass` action skips to the +next [tier](tier.mdx), to enable hierarchical security policy. + +For `kubectl` [commands](https://kubernetes.io/docs/reference/kubectl/overview/), the following case-insensitive aliases +may be used to specify the resource type on the CLI: +`stagednetworkpolicy.projectcalico.org`, `stagednetworkpolicies.projectcalico.org` and abbreviations such as +`stagednetworkpolicy.p` and `stagednetworkpolicies.p`. + +## Sample YAML + +This sample policy allows TCP traffic from `frontend` endpoints to port 6379 on +`database` endpoints. + +```yaml +apiVersion: projectcalico.org/v3 +kind: StagedNetworkPolicy +metadata: + name: internal-access.allow-tcp-6379 + namespace: production +spec: + tier: internal-access + selector: role == 'database' + types: + - Ingress + - Egress + ingress: + - action: Allow + protocol: TCP + source: + selector: role == 'frontend' + destination: + ports: + - 6379 + egress: + - action: Allow +``` + +## Definition + +### Metadata + +| Field | Description | Accepted Values | Schema | Default | +| --------- | ------------------------------------------------------------------ | --------------------------------------------------- | ------ | --------- | +| name | The name of the network policy. Required. | Alphanumeric string with optional `.`, `_`, or `-`. | string | | +| namespace | Namespace provides an additional qualification to a resource name. | | string | "default" | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------- | --------------------- | --------------------------------------------- | +| order | Controls the order of precedence. $[prodname] applies the policy with the lowest value first. | | float | | +| tier | Name of the [tier](tier.mdx) this policy belongs to. | | string | `default` | +| selector | Selects the endpoints to which this policy applies. | | [selector](#selector) | all() | +| types | Applies the policy based on the direction of the traffic. To apply the policy to inbound traffic, set to `Ingress`. To apply the policy to outbound traffic, set to `Egress`. To apply the policy to both, set to `Ingress, Egress`. | `Ingress`, `Egress` | List of strings | Depends on presence of ingress/egress rules\* | +| ingress | Ordered list of ingress rules applied by policy. | | List of [Rule](#rule) | | +| egress | Ordered list of egress rules applied by this policy. | | List of [Rule](#rule) | | +| serviceAccountSelector | Selects the service account(s) to which this policy applies. Select a specific service account by name using the `projectcalico.org/name` label. | | [selector](#selector) | all() | +| performanceHints | Contains a list of hints to Calico's policy engine to help process the policy more efficiently. Hints never change the enforcement behaviour of the policy. The available hints are described [below](#performance-hints). | `AssumeNeededOnEveryNode` | List of strings | | + +\* If `types` has no value, $[prodname] defaults as follows. + +> | Ingress Rules Present | Egress Rules Present | `Types` value | +> | --------------------- | -------------------- | ----------------- | +> | No | No | `Ingress` | +> | Yes | No | `Ingress` | +> | No | Yes | `Egress` | +> | Yes | Yes | `Ingress, Egress` | + +### Rule + + + +### ICMP + + + +### EntityRule + + + +### Selector + + + +### Ports + + + +### ServiceAccountMatch + + + +### ServiceMatch + + + +### Performance Hints + +Performance hints provide a way to tell $[prodname] about the intended use of the policy so that it may +process it more efficiently. Currently only one hint is defined: + +* `AssumeNeededOnEveryNode`: normally, $[prodname] only calculates a policy's rules and selectors on nodes where + the policy is actually in use (i.e. its selector matches a local endpoint). This saves work in most cases. + The `AssumeNeededOnEveryNode` hint tells $[prodname] to treat the policy as "in use" on *every* node. This is + useful for large policy sets that are known to apply to all (or nearly all) endpoints. It effectively "preloads" + the policy on every node so that there is less work to do when the first endpoint matching the policy shows up. + It also prevents work from being done to tear down the policy when the last endpoint is drained. + +## Supported operations + +| Datastore type | Create/Delete | Update | Get/List | Notes | +| ------------------------ | ------------- | ------ | -------- | ----- | +| Kubernetes API datastore | Yes | Yes | Yes | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/tier.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/tier.mdx new file mode 100644 index 0000000000..a32a0d49ca --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/tier.mdx @@ -0,0 +1,62 @@ +--- +description: Reference for the Tier resource in Calico Cloud connected clusters that groups tiered policies into ordered evaluation buckets. +--- + +# Tier + +A tier resource (`Tier`) represents an ordered collection of [NetworkPolicies](networkpolicy.mdx) +and/or [GlobalNetworkPolicies](globalnetworkpolicy.mdx). +Tiers are used to divide these policies into groups of different priorities. These policies +are ordered within a Tier: the additional hierarchy of Tiers provides more flexibility +because the `Pass` `action` in a Rule jumps to the next Tier. Some example use cases for this are: + +- Allowing privileged users to define security policy that takes precedence over other users. +- Translating hierarchies of physical firewalls directly into $[prodname] policy. + +For `kubectl` [commands](https://kubernetes.io/docs/reference/kubectl/overview/), the following case-insensitive aliases +may be used to specify the resource type on the CLI: +`tier.projectcalico.org`, `tiers.projectcalico.org` and abbreviations such as +`tier.p` and `tiers.p`. + +## How policy is evaluated + +When a new connection is processed by $[prodname], each tier that contains a policy that applies to the endpoint processes the packet. +Tiers are sorted by their `order` - smallest number first. + +Policies in each Tier are then processed in order. + +- If a [NetworkPolicy](networkpolicy.mdx) or [GlobalNetworkPolicy](globalnetworkpolicy.mdx) in the Tier `Allow`s or `Deny`s the packet, then evaluation is done: the packet is handled accordingly. +- If a [NetworkPolicy](networkpolicy.mdx) or [GlobalNetworkPolicy](globalnetworkpolicy.mdx) in the Tier `Pass`es the packet, the next Tier containing a Policy that applies to the endpoint processes the packet. + +If the Tier applies to the endpoint, but takes no action on the packet the packet is dropped. This behaviour can be changed by setting the `defaultAction` of a tier to `Pass`. + +If the last Tier applying to the endpoint `Pass`es the packet, that endpoint's [Profiles] (profile.mdx) are evaluated. + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: Tier +metadata: + name: internal-access +spec: + order: 100 + defaultAction: Deny +``` + +## Definition + +### Metadata + +| Field | Description | Accepted Values | Schema | +| ----- | --------------------- | --------------- | ------ | +| name | The name of the tier. | | string | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------ | --------------- | ------ | --------------------- | +| order | (Optional) Indicates priority of this Tier, with lower order taking precedence. No value indicates highest order (lowest precedence) | | float | `nil` (highest order) | +| defaultAction | (Optional) Indicates the default action, when this Tier applies to an endpoint, but takes no action on the packet | `Deny`, `Pass` | string | `Deny` | + +All Policies created by $[prodname] orchestrator integrations are created in the default (last) Tier. diff --git a/calico-cloud_versioned_docs/version-24-1/reference/resources/workloadendpoint.mdx b/calico-cloud_versioned_docs/version-24-1/reference/resources/workloadendpoint.mdx new file mode 100644 index 0000000000..1baaa05fa3 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/resources/workloadendpoint.mdx @@ -0,0 +1,141 @@ +--- +description: Reference for the WorkloadEndpoint resource in Calico Cloud connected clusters that represents a pod or VM interface for policy and IPAM. +--- + +# Workload endpoint + +import Ipnat from '@site/calico-cloud_versioned_docs/version-24-1/_includes/content/_ipnat.mdx'; + +A workload endpoint resource (`WorkloadEndpoint`) represents an interface +connecting a $[prodname] networked container or VM to its host. + +Each endpoint may specify a set of labels and list of profiles that $[prodname] will use +to apply policy to the interface. + +A workload endpoint is a namespaced resource, that means a +[NetworkPolicy](networkpolicy.mdx) +in a specific namespace only applies to the WorkloadEndpoint in that namespace. +Two resources are in the same namespace if the namespace value is set the same +on both. + +This resource is not supported in `kubectl`. + +:::note + +While `calicoctl` allows the user to fully manage Workload Endpoint resources, +the lifecycle of these resources is generally handled by an orchestrator-specific +plugin such as the $[prodname] CNI plugin. In general, we recommend that you only +use `calicoctl` to view this resource type. + +::: + +**Multiple networks** + +If multiple networks are enabled, workload endpoints will have additional labels which can be used in network policy selectors: + +- `projectcalico.org/network`: The name of the network specified in the NetworkAttachmentDefinition. +- `projectcalico.org/network-namespace`: This namespace the network is in. +- `projectcalico.org/network-interface`: The network interface for the workload endpoint. + +For more information, see the [multiple-networks how-to guide](../../networking/configuring/multiple-networks.mdx). + +## Sample YAML + +```yaml +apiVersion: projectcalico.org/v3 +kind: WorkloadEndpoint +metadata: + name: node1-k8s-my--nginx--b1337a-eth0 + namespace: default + labels: + app: frontend + projectcalico.org/namespace: default + projectcalico.org/orchestrator: k8s +spec: + node: node1 + orchestrator: k8s + endpoint: eth0 + containerID: 1337495556942031415926535 + pod: my-nginx-b1337a + endpoint: eth0 + interfaceName: cali0ef24ba + mac: ca:fe:1d:52:bb:e9 + ipNetworks: + - 192.168.0.0/32 + profiles: + - profile1 + ports: + - name: some-port + port: 1234 + protocol: TCP + - name: another-port + port: 5432 + protocol: UDP +``` + +## Definitions + +### Metadata + +| Field | Description | Accepted Values | Schema | Default | +| --------- | ------------------------------------------------------------------ | -------------------------------------------------- | ------ | --------- | +| name | The name of this workload endpoint resource. Required. | Alphanumeric string with optional `.`, `_`, or `-` | string | | +| namespace | Namespace provides an additional qualification to a resource name. | | string | "default" | +| labels | A set of labels to apply to this endpoint. | | map | | + +### Spec + +| Field | Description | Accepted Values | Schema | Default | +| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------- | ---------------------------------------------- | ------- | +| workload | The name of the workload to which this endpoint belongs. | | string | | +| orchestrator | The orchestrator that created this endpoint. | | string | | +| node | The node where this endpoint resides. | | string | | +| containerID | The CNI CONTAINER_ID of the workload endpoint. | | string | | +| pod | Kubernetes pod name for this workload endpoint. | | string | | +| endpoint | Container network interface name. | | string | | +| ipNetworks | The CIDRs assigned to the interface. | | List of strings | | +| ipNATs | List of 1:1 NAT mappings to apply to the endpoint. | | List of [IPNATs](#ipnat) | | +| awsElasticIPs | List of AWS Elastic IP addresses that should be considered for this workload; only used for workloads in an AWS-backed IP pool. This should be set via the `cni.projectcalico.org/awsElasticIPs` Pod annotation. | | List of valid IP addresses | | +| ipv4Gateway | The gateway IPv4 address for traffic from the workload. | | string | | +| ipv6Gateway | The gateway IPv6 address for traffic from the workload. | | string | | +| profiles | List of profiles assigned to this endpoint. | | List of strings | | +| interfaceName | The name of the host-side interface attached to the workload. | | string | | +| mac | The source MAC address of traffic generated by the workload. | | IEEE 802 MAC-48, EUI-48, or EUI-64 | | +| ports | List on named ports that this workload exposes. | | List of [WorkloadEndpointPorts](#endpointport) | | + +### IPNAT + + + +### EndpointPort + +A WorkloadEndpointPort associates a name with a particular TCP/UDP/SCTP port of the endpoint, allowing it to +be referenced as a named port in [policy rules](networkpolicy.mdx#entityrule). + +| Field | Description | Accepted Values | Schema | Default | +| -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------- | ------ | ------- | +| name | The name to attach to this port, allowing it to be referred to in [policy rules](networkpolicy.mdx#entityrule). Names must be unique within an endpoint. | | string | | +| protocol | The protocol of this named port. | `TCP`, `UDP`, `SCTP` | string | | +| port | The workload port number. | `1`-`65535` | int | | +| hostPort | Port on the host that is forwarded to this port. | `1`-`65535` | int | | +| hostIP | IP address on the host on which the hostPort is accessible. | `1`-`65535` | int | | + +:::note + +On their own, WorkloadEndpointPort entries don't result in any change to the connectivity of the port. +They only have an effect if they are referred to in policy. + +::: + +:::note + +The hostPort and hostIP fields are read-only and determined from Kubernetes hostPort configuration. +These fields are used only when host ports are enabled in Calico. + +::: + +## Supported operations + +| Datastore type | Create/Delete | Update | Get/List | Notes | +| --------------------- | ------------- | ------ | -------- | -------------------------------------------------------- | +| Kubernetes API server | No | Yes | Yes | WorkloadEndpoints are directly tied to a Kubernetes pod. | diff --git a/calico-cloud_versioned_docs/version-24-1/reference/rest-api-reference.mdx b/calico-cloud_versioned_docs/version-24-1/reference/rest-api-reference.mdx new file mode 100644 index 0000000000..42e7e5e6cb --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/reference/rest-api-reference.mdx @@ -0,0 +1,15 @@ +--- +description: REST API reference index for Calico Cloud covering the SaaS management plane endpoints used by the Calico Cloud web console. +--- + +# REST API Reference + +import SwaggerUI from 'swagger-ui-react'; + + + + diff --git a/calico-cloud_versioned_docs/version-24-1/release-notes/index.mdx b/calico-cloud_versioned_docs/version-24-1/release-notes/index.mdx new file mode 100644 index 0000000000..0179651b2f --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/release-notes/index.mdx @@ -0,0 +1,1491 @@ +--- +description: Find what changed in each release of Calico Cloud, including new features, web console updates, enhancements, and fixes across dated releases. +title: Calico Cloud release notes +sidebar_label: Release notes +--- + +import IconUser from '/img/icons/user-icon.svg'; + +# Calico Cloud release notes + +

October 13, 2026 (version 24.0.0)

+ +### New features and enhancements + +#### Policy editor improvements + +The policy editor in the web console has a redesigned layout and a new YAML view. +You can switch between the form and the YAML for a policy while you create or edit it. + +For more information, see [Network policy tutorial](../network-policy/policy-tiers/policy-tutorial-ui.mdx). + +#### Gateway WAF for routes and namespaces (tech preview) + +You can now attach web application firewall (WAF) protection to an individual Gateway or HTTPRoute behind Calico Ingress Gateway, instead of to every gateway at once, and tune it for each namespace. +Platform teams set a global baseline, application teams tune it for their own namespaces and routes, and you can add custom rules as plugins. + +For more information, see [Gateway WAF](../threat/gateway-waf/overview.mdx). + +#### New L7 log collectors + +$[prodname] can now collect L7 logs with an eBPF collector that captures HTTP and TLS metadata in the kernel, with no proxy or sidecar in the data path. +You enable it once for the cluster, and it works with any data plane. +If you run Istio Ambient Mode, you can also collect L7 logs from waypoint proxies with the new Waypoint collector. + +For more information, see [L7 logs](../observability/elastic/l7/overview.mdx). + +#### Operator metrics + +The Tigera Operator now exposes Prometheus metrics for component status and TLS certificate expiry. +You can use them to alert on degraded components and on TLS certificates that are about to expire. + +For more information, see [Operator metrics](../operations/monitor/metrics/operator-metrics.mdx). + +#### Selector-scoped Felix configuration (tech preview) + +You can now apply a `FelixConfiguration` resource to a set of nodes that you choose with a label selector. + +For more information, see [Felix configuration](../reference/resources/felixconfig.mdx#selector-scoped-configuration). + +#### Versioned Helm charts + +The $[prodname] Helm charts are now released with every $[prodname] release, and you can install a chart by its release version, such as `v24.0.0`. +You still need to set `installer.calicoCloudVersion` when you install with Helm. + +### Deprecated and removed features + +* The compliance reporting feature is removed from the $[prodname] web console. + The compliance reporting docs, including the `GlobalReport` resource reference, are removed. +* The Envoy collector for L7 logs is deprecated. + Use the eBPF collector or the Waypoint collector instead. +* $[prodname] 21 is no longer supported. + +### Updating + +* ***Potential breaking change:*** Calico Ingress Gateway now runs each gateway's proxy in the same namespace as its `Gateway` resource, and the gateway controller moves to the `calico-system` namespace. + When you upgrade, existing proxies move out of the `tigera-gateway` namespace into the namespace of each `Gateway`. + Anything that refers to `tigera-gateway` must move with them, including network policy, monitoring, RBAC, and external DNS. + Merged gateways (`mergeGateways: true`) are no longer supported, so each gateway gets its own load balancer. + Migrate in this order: + 1. Before you upgrade, if you use a global default deny policy, create a network policy in each `Gateway` namespace that allows the proxy pods, so they can start as soon as they move. + For the policy, see [Create an ingress gateway](../networking/ingress-gateway/create-ingress-gateway.mdx). + 2. Upgrade your cluster. + For more information, see [Upgrade $[prodname]](../get-started/upgrade-cluster.mdx). + 3. After the upgrade, point monitoring, RBAC, and external DNS at each `Gateway` namespace instead of `tigera-gateway`. + If you used merged gateways, plan for one load balancer, DNS record, and certificate for each gateway. +* If you install from a private registry with the latest Helm chart, add the `cc-operator` image that matches the version you're installing or upgrading to into your registry. + For more information, see [Set up a private registry](../get-started/setup-private-registry.mdx). + +### Bug fixes + +* Fixed the default `cc-operator` image path in the Helm chart when you configure a private registry. + Previously, if you didn't set `imagePath`, the chart used `calico-cloud-public` as the image path. + If you changed your private registry to put `cc-operator` under a `calico-cloud-public` path to work around this, update your registry to match the image path you set. + +### Known issues + +* You can't connect a Kubernetes 1.36 cluster to $[prodname] 24.0.0 if Calico Open Source on that cluster was installed with the `projectcalico.org/v3` CRDs. + The installation needs the `admissionregistration.k8s.io/v1beta1` API, which Kubernetes 1.36 doesn't provide. + +

August 28, 2026 (versions 23.0.2 and 22.5.1)

+ +Versions 23.0.2 and 22.5.1 are patch releases that bring the same fix to $[prodname] 23 and 22. + +### Bug fixes + +* Fixed an issue where a change to how Azure Kubernetes Service (AKS) manages Calico could stop an installation from finishing, or stop the operator from managing resources on a cluster you already have. + $[prodname] now detects the changed configurations, so installations complete and existing clusters keep running. +* Version 23.0.2 only. Fixed an issue where the policy board in the web console returned an error when one of your own default-deny policies covered the `calico-system` namespace. + $[prodname] now includes a policy that allows traffic from the API server to Guardian, so the board loads without you having to add a policy of your own. + +

August 26, 2026 (web console update)

+ +### New features and enhancements + +#### Dark mode + +The web console now has a dark theme. +To switch themes, click the sun or moon icon in the top-right corner. + +

August 14, 2026 (version 23.0.1)

+ +### Bug fixes + +* Fixed a problem introduced in version 23.0.0 that denied cluster DNS traffic during an upgrade to version 23. + Clusters upgrading from any earlier version of $[prodname] could hit it. + The installer now corrects the cluster DNS policy while the upgrade runs, so $[prodname] components keep resolving names, Guardian connects without crash looping, and the upgrade finishes. + +### Updating + +* ***Breaking change:*** The `allow-tigera` tier is renamed to `calico-system`. + The Tigera Operator moves its own policies into the new tier during the upgrade, but it does not update resources that you created. + Because a policy name is prefixed with the name of its tier, the policies in the tier are renamed with it, from `allow-tigera.` to `calico-system.`. + Before you upgrade, review anything that refers to the tier or to those policy names, including: + * Network policies that you added to the `allow-tigera` tier. + * RBAC rules that name `allow-tigera` or `allow-tigera.*` in `resourceNames`. + * Manifests, scripts, and automation that reference policies by their `allow-tigera.` names. + * Dashboards, alerts, and queries that filter on `tier="allow-tigera"` or on policy names. + + For more information, see [Change calico-system tier behavior](../network-policy/policy-tiers/calico-system.mdx). + +

July 28, 2026 (version 23.0.0)

+ +### New features and enhancements + +#### Programmatic access to usage data + +You can now create a read-only API key in the web console and use it to call the $[prodname] Usage API. +The API returns the same vCPU usage and managed cluster data shown on the Usage Metrics page, so you can pull it into your own capacity planning, FinOps, and license-compliance workflows. + +For more information, see [Access usage data programmatically](../operations/usage-api.mdx). + +#### Maglev load balancing for services + +$[prodname] now supports Maglev consistent-hash load balancing for external traffic to services. +Maglev lets your nodes act as a distributed, horizontally scalable load balancer, so external connections can fail over to another node when a node goes down. +This is useful when you advertise service IPs to your network as ECMP routes. +You enable Maglev per service with an annotation. + +For more information, see [Add Maglev load balancing to a service](../networking/configuring/add-maglev-load-balancing.mdx). + +#### L2 reachability for pods and services without BGP + +You can now make pod IPs and LoadBalancer service VIPs reachable on a flat layer 2 network without BGP or an overlay. +When you enable `localSubnetL2Reachability`, $[prodname] runs a userspace ARP/NDP responder that answers for local pod IPs and LoadBalancer VIPs that share the host subnet. +This is useful for on-premises, edge, and other environments where you can't configure the upstream router. + +For more information, see [L2 reachability for pods and services without BGP](../networking/configuring/local-subnet-l2-reachability.mdx). + +#### Host IP support for egress gateways + +You can now configure egress gateways to use the host node's IP as the source address for outbound traffic, instead of the gateway's pod IP. +This gives external firewalls and services a stable, well-known set of source IPs to allowlist, and lets many application pods share a small fixed set of gateways. +Host IP mode is most useful for on-premises clusters where pod IPs aren't routable outside the cluster. + +For more information, see [Configure egress gateways with host IP support](../networking/egress/egress-gateway-host-ip.mdx). + +#### Load balancer node maintenance + +You can now mark a node for maintenance so the eBPF data plane stops sending new service traffic to its pods while existing connections finish. +This lets you drain nodes for upgrades, repairs, or capacity changes without dropping in-flight sessions or stateful workloads. +This feature is supported on the eBPF data plane only. + +For more information, see [Mark a load balancer node for maintenance](../networking/configuring/mark-lb-node-for-maintenance.mdx). + +#### Support for Kubernetes 1.36 + +This release adds support for Kubernetes 1.36. + +### Deprecated and removed features + +* Image Assurance, Container Threat Detection, and Kibana are removed in this release. + These features were previously deprecated and are no longer available in $[prodname] 23. + +### Updating + +* ***Potential breaking change:*** If you have dependencies on the `allow-tigera` tier those dependencies will need to be updated. + $[prodname] moves its own policies into the new `calico-system` tier during the upgrade, but it does not update resources that you created. + Before you upgrade, review anything that refers to the tier by name, including: + * Network policies that you added to the `allow-tigera` tier, these must be removed before starting the upgrade. + * If you do not remove your network policies from the `allow-tigera` tier, the install will block until they are removed. + * Tiers that you ordered relative to `allow-tigera`, the new `calico-system` tier is the same order as `allow-tigera`. + * RBAC rules that name `allow-tigera` in `resourceNames`. + * Dashboards, alerts, and queries that filter on `tier="allow-tigera"`. + + **Recommendation:** Do not add policies to the $[prodname] tier, if changes are needed create a separate tier and add your policies there. + + For more information, see [Change calico-system tier behavior](../network-policy/policy-tiers/calico-system.mdx). + +### Known issues + +Upgrading to version 23.0.0 from any earlier version of $[prodname] can deny cluster DNS traffic and leave the upgrade unfinished. +The cluster DNS policy moves into the new `calico-system` tier while the `allow-tigera` tier is still in place, so $[prodname] components lose name resolution partway through the upgrade. + +Version 23.0.0 is no longer available to install. +Upgrade to [version 23.0.1](#23.0.1) instead, which fixes the problem. + +If a cluster upgraded to version 23.0.0 and hit this problem, upgrading it to version 23.0.1 does not repair it. +[Contact Support](../get-help/support.mdx) to return the cluster to a healthy state first, and then upgrade it. + +### Updating + +* ***Breaking change:*** The `allow-tigera` tier is renamed to `calico-system`. + The Tigera Operator moves its own policies into the new tier during the upgrade, but it does not update resources that you created. + Because a policy name is prefixed with the name of its tier, the policies in the tier are renamed with it, from `allow-tigera.` to `calico-system.`. + Before you upgrade, review anything that refers to the tier or to those policy names, including: + * Network policies that you added to the `allow-tigera` tier. + * RBAC rules that name `allow-tigera` or `allow-tigera.*` in `resourceNames`. + * Manifests, scripts, and automation that reference policies by their `allow-tigera.` names. + * Dashboards, alerts, and queries that filter on `tier="allow-tigera"` or on policy names. + + For more information, see [Change calico-system tier behavior](../network-policy/policy-tiers/calico-system.mdx). + +

June 22, 2026 (version 22.5.0)

+ +### New features and enhancements + +* Security Updates + +

June 9, 2026 (version 22.4.1)

+ +### New features and enhancements + +* Adds support for Kubernetes 1.35. + +

May 19, 2026 (version 22.4.0)

+ +### New features and enhancements + +#### Audit logs + +$[prodname] now records audit events for user activity, including user sign-ins and group assignments. +To view them in the web console, go to **Activity > Audit logs**. + +#### Enhancements + +* The **Usage Metrics** page now includes a per-project breakdown of vCPU hours. + +

April 7, 2026 (version 22.3.0)

+ +### New features and enhancements + +#### Notifications for cluster connections + +Administrators can now set up notifications to receive alerts when managed clusters connect to or disconnect from $[prodname]. +This helps platform teams stay informed about cluster availability and take action quickly when clusters become disconnected or unhealthy. + +To configure notifications, click the user icon **> Notifications**. + +

March 5, 2026 (version 22.2.0)

+ +### New features and enhancements + +#### Istio Ambient Mode (tech preview) + +$[prodname] now provides a bundled version of Istio in ambient mode, a sidecarless architecture that delivers robust mTLS encryption and service mesh security while significantly reducing resource consumption and operational overhead. +This implementation, managed by the Tigera Operator, features an enhanced zTunnel proxy that preserves original destination ports so existing Calico and Kubernetes network policies continue to function seamlessly without requiring rewrites. + +For more information, see [Istio Ambient Mode](../compliance/istio/about-istio-ambient.mdx). + +#### Enhancements + +* Enhancements to Calico Cloud dashboards. + +

February 5, 2026 (web console update)

+ +### New features and enhancements + +#### Projects + +This release introduces self-service Projects, allowing you to organize managed clusters into logical groups based on team, environment, or geographic location. +You can now create up to three projects, select hosting regions, and scope observability tools to specific project environments. + +For more information, see [Projects](../tutorials/calico-cloud-features/projects.mdx). + +#### Namespaced access to dashboards + +Administrators can now provide users with namespaced access to dashboards. +This ensures that users view logs and information only for the specific areas they are authorized to manage. +Namespaced access to dashboards can be managed by specifying namespaces in the **View Network Sets** custom role permission. + +:::important +Changes in this release may cause some users to lose access to dashboard data. +To restore access, administrators must edit users' custom roles to add the **View Global Network Sets** permission or the **View Network Sets** permission with the appropriate namespaces. +::: + +For more information, see [Create and assign custom roles](../users/create-and-assign-custom-roles.mdx). + +### Deprecated and removed features + +* Kibana dashboards are deprecated and will be removed in an upcoming release. + During the deprecation period, you will have read-only access to Kibana dashboards. + You can still [create custom dashboards](../observability/create-custom-dashboard.mdx) using Calico Cloud's built-in dashboards. + +

December 1, 2025 (version 22.1.0)

+ +### New features and enhancements + +#### Dashboards are GA + +Calico Cloud's built-in observability dashboards are now generally available. +These integrated views provide cluster operators with a detailed view of network operations, including metrics on traffic flows, policy decisions, and layer 7 (HTTP) requests. +You can supplement the default dashboards by creating your own custom dashboards to suit your specific needs. + +For more information, see [Dashboards](../observability/dashboards.mdx). + +#### Enhancements + +* Added various user experience improvements to dashboards in the web console. + +### Bug fixes + +* Fixed a problem with time range filters for packet captures in Service Graph. +* Fixed a bug that caused incorrect operands to appear when you added ports to a network policy in the web console. + +#### Support for Kubernetes 1.34 + +This release adds support for Kubernetes 1.34. + +### Known issues + +:::warning[breaking change for windows nodes] +Pod networking fails if you install or upgrade to Calico Cloud 22.1.0 on a cluster that has the following: +* The cluster has Windows nodes. +* The cluster's `IPPool` resource has `spec.encapsulation` set to `None`. + +Updating a cluster with Windows nodes and no encapsulation causes pod networking to fail on Windows nodes. + +This issue will be fixed in an upcoming release. +::: + +

October 21, 2025 (web console update)

+ +### New features and enhancements + +* Improved search and user experience for the **Policies** page in the Calico Cloud Free Tier web console. + +### Bug fixes + +* Added various user experience fixes and improvements to dashboards in the web console. + +

October 6, 2025 (version 22.0.1)

+ +### Bug fixes + +* Removed an unnecessary impersonate permission for the calico-guardian ClusterRole. + The guardian now uses direct permissions instead of impersonation, improving cluster security. + +

September 8, 2025 (version 22.0.0)

+ +### New features and enhancements + +#### Calico Ingress Gateway is GA + +Calico Ingress Gateway is now supported with general availability. +Calico Ingress Gateway is an enterprise-hardened, 100% upstream distribution of Envoy Gateway. Envoy Gateway is an implementation of the Kubernetes Gateway API with several extensions that provide advanced security and traffic management features. + +For more information, see [Configure an ingress gateway](../networking/ingress-gateway/about-calico-ingress-gateway.mdx). + +#### External load balancers for Calico Ingress Gateway + +This release adds customization options for specifying external load balancers for `Gateway` resources in your cluster. + +For more information, see [Customize gateway deployment and features](../networking/ingress-gateway/about-calico-ingress-gateway.mdx). + +#### Web application firewall for Calico Ingress Gateway + +The release adds the ability to configure a web application firewall for Calico Ingress Gateway. + +For more information, see [Deploying WAF with an ingress gateway](../threat/deploying-waf-ingress-gateway.mdx). + +#### Enhanced visibility for denied pre-DNAT traffic + +This update delivers flow logs specifically for denied traffic by Calico pre-DNAT policies. +Now, you can quickly identify and understand why certain traffic is being blocked at the earliest stage of packet processing, greatly simplifying troubleshooting and providing assurance that your pre-DNAT policies are effectively enforcing your desired access controls. + +For more information, see [Apply policy to forwarded traffic](../network-policy/hosts/host-forwarded-traffic.mdx). + +#### HTTP header-based matching for application layer policies + +This release includes support for HTTP header-based matching for application layer policies. +You can now write deny and allow rules for L7 ingress traffic by matching values in HTTP headers. + +For more information, see [Global network policy](../reference/resources/globalnetworkpolicy.mdx#httpheadermatch). + +#### Enhancements + +* To support a minimal footprint and simplify resource management, the API server component and its associated resources have been moved from the `tigera-system` namespace to the `calico-system` namespace. + +### Version support + +You can now install or upgrade to the following versions: + +* Calico Cloud 22 +* Calico Cloud 21 +* Calico Cloud 20 + +{/* ### Bug fixes */} + +

August 5, 2025 (web console update)

+ +### New features and enhancements + +#### Predefined user roles for Entra ID groups + +We added support for granting predefined role permissions to Entra ID security groups. +This allows platform administrators to bulk assign access to the Calico Cloud web console without having to create custom roles. + +For more information, see [Give role-based access to an Entra ID group](../users/create-custom-role-for-entra-id-group.mdx). + +

June 23, 2025 (version 21.3.0)

+ +### New features and enhancements + +#### Policy recommendations for Calico Cloud Free Tier + +We added support for policy recommendations for Calico Cloud Free Tier. +The policy recommendations tool provides a set of recommended network policies based on the traffic patterns in your cluster. +When you get a recommendation, you can review, stage, and test the policy before applying it to your cluster. + +For more information, See [Policy recommendations](../free/overview.mdx#policy-recommendations). + +#### Support for WireGuard encryption between clusters + +We added support for WireGuard encryption between federated services and endpoints in different clusters. + +For more information, see [Creating the cluster mesh](../multicluster/kubeconfig.mdx). + + +#### Improvements to flow log reporting for staged network policies + +This release introduces changes to improve how staged network policies are reported in flow logs. +Previously, a flow log reported the action of staged network policy rules at the time a connection was initiated. +For long-lived connections, changing a staged policy did not affect the reported action. +Now, flow logs report the action that represents the current policy rules. +Flow logs report an action that reflects how a new connection would interact with the current staged policies. + +As part of this, we've also added more granular information about policies in the flow logs. +For more information, see [Flow log data types](../observability/elastic/flow/datatypes.mdx). + +#### Security event webhooks for Alertmanager + +We added support for using webhooks to post security alerts directly to Alertmanager. + +For more information, see [Webhooks for security event alerts](../threat/configuring-webhooks.mdx). + +#### View rule details for Web Application Firewall + +You can now use the web console to view details of the default rule set used by Web Application Firewall. +From the **Web Application Firewall** page, click the **Rulesets** tab to open a list of default rules. + +#### Enhancements + +* Added web console support for `AdminNetworkPolicy` and `BaseAdminNetworkPolicy` tiers (view-only). +* Performance and user experience improvements to custom dashboards. + +#### Support for Kubernetes 1.32 and 1.33 + +This release adds support for Kubernetes 1.32 and 1.33. + +### Known issues + +* If your managed cluster includes nodes running Bottlerocket OS, you may encounter installation errors and network connectivity issues when updating to $[prodname] 21.3.0. + + To prevent or resolve these issues, modify the Felix configuration resource by setting the `dnsPolicyMode` parameter to `DelayDeniedPacket`. Apply this patch either before or after updating to $[prodname] 21.3.0: + + ```bash + kubectl patch --type json felixconfigurations.projectcalico.org default -p '[{"op":"add","path":"/spec/dnsPolicyMode","value":"DelayDeniedPacket"}]' + ``` + + This issue is caused by a new default value for the `dnsPolicyMode` parameter in the Felix configuration. + In version 21.3.0, if not explicitly set, `dnsPolicyMode` defaults to `Inline`, which is incompatible with Bottlerocket OS and leads to the described problems. + + This issue will be addressed in an upcoming release. + +## May 5, 2025 (web console update) + +### New features and enhancements + +#### Introducing Calico Cloud Free Tier + +We are excited to introduce Calico Cloud Free Tier, a new offering designed to provide users of Calico Open Source with powerful observability tools at no cost. +If you're running Calico Open Source, you can now connect a cluster to our cloud-based web console to visualize network traffic, understand policy interactions, and gain essential insights into your Kubernetes networking. + +Calico Cloud Free Tier offers a selection of core observability features from the full Calico Cloud platform, focused on helping you understand what's happening within your Calico Open Source cluster. +It leverages Calico's flow logs API (introduced in Calico Open Source 3.30) to securely stream aggregated flow log data from your cluster to the Calico Cloud console. + +Key features include: +* Dashboards. Access pre-built dashboards visualizing traffic volume, DNS activity, and network performance based on the last 24 hours of flow data. +* Service Graph. Get a dynamic, topographical map of service-to-service communication and endpoint flows within your cluster, making it easy to visualize connections and troubleshoot issues. +* Policy board. Inspect your network policies and staged network policies to make sure they're working as you expect. + +To get started, [sign up for a free Calico Cloud account](https://calicocloud.io). + +For more information about Calico Cloud Free Tier, see the [overview](../free/overview.mdx) or the [quickstart guide](../free/quickstart.mdx). + +

April 29, 2025 (version 21.2.0)

+ +### New features and enhancements + +#### New pre-defined user roles + +We added two new pre-defined user roles to help you manage access to Calico Cloud: +* The *User Admin* role has the ability to manage team members and their assigned roles. +* The *Cluster Connection Admin* role has the ability to manage cluster connections and their settings. + +For more information, see [Set up users](../users/user-management.mdx). + +#### Improvements to Dashboards + +This release includes various improvements to the look and performance of the dashboards. +Several dashboards have been renamed to better reflect their content. +* The Data Volume dashboard is now called the Traffic Volume dashboard. +* The L7 dashboard is now called the HTTP Traffic dashboard. +* The TCP Performance dashboard is now called the Network Performance dashboard. + +For more information, see [Dashboards](../observability/dashboards.mdx). + +

April 1, 2025 (version 21.1.0)

+ +### Updated pricing structure + +We've updated our pricing structure to make it easier to understand and manage. +Now we calculate usage based on your vCPU hours, rather than node hours. +And we stopped charging for log data ingestion. + +When you upgrade, you'll notice a few changes: +* The previous metrics (node hours and ingested log data) are gone. +* The historical consumption data will continue to show your older metrics. + +For more information, see [Usage and billing](../operations/usage-metrics.mdx). + +### Changes to version availability + +Beginning with this release, users will not have access to versions of Calico Cloud that are older than the latest version at the time they joined. +For example, if you start using Calico Cloud with version 21.1.0, you will not be able to install version 19.4.0 or version 20.0.0. +As new versions are released, you will have access to all versions released after the version you joined with. + +### Deprecated and removed features + +The Image Assurance and container threat detection features are deprecated and will be removed in a future release. +Not all users will be able to use these deprecated features from this version onward: + +* For users who start using Calico Cloud in April 2025 or later, these feature are not available. +* Legacy users, who started using Calico Cloud before April 2025, can continue to use these features until they are removed in a future release. + +This split availability applies also to the compliance reports feature, which was deprecated in release 20.3.0. + +### Upgrade instructions for users of deprecated features + +By default, new installations and upgrades to Calico Cloud 21.1.0 will disable Image Assurance, container threat detection, and compliance reporting. + +Additional steps are required to enable these features for legacy users who started using Calico Cloud before April 2025. + +For more information, see [Install Calico Cloud](../get-started/install-cluster.mdx) and [Upgrade Calico Cloud](../get-started/upgrade-cluster.mdx). + +

March 10, 2025 (version 21.0.0)

+ +### New features and enhancements + +#### Custom dashboards (tech preview) + +You can now build your own dashboard visualizations by creating new charts and queries in a custom dashboard. +For simple changes, you can import one of the many standard charts and make edits to the query parameters as needed. +Advanced users can create custom charts from scratch to display information from our flow, DNS, and L7 logs. + +For more information, see [Create a custom dashboard](../observability/create-custom-dashboard.mdx). + +#### Introducing Calico Ingress Gateway (tech-preview) + +Calico Cloud now includes the ability to deploy Calico Ingress Gateway which is an Enterprise hardened, 100% upstream distribution of Envoy Gateway. Envoy Gateway is an implementation of the Kubernetes Gateway API with several extensions that provide advanced security and traffic management features. + +For more information, see [Configure an ingress gateway](../networking/ingress-gateway/about-calico-ingress-gateway.mdx). + +#### IPAM for load balancers + +Calico Cloud now extends its IPAM capabilities to support service LoadBalancer IP allocation, providing a centralized, automated approach to managing LoadBalancer IPs within Kubernetes clusters. + +For more information, see [LoadBalancer IP address management](../networking/ipam/service-loadbalancer.mdx). + +#### Enhancements + +* We improved how the web console performs in cases where there are a large number of policy tiers for managed clusters. +To see these optimizations, make sure your managed clusters are running Calico Cloud 21.0.0 or later. + +Calico Cloud versions 18 and earlier are no longer supported. + +For information about upgrading, see [Upgrade Calico Cloud](../get-started/upgrade-cluster.mdx). + +### Deprecated and removed features + +* The admission controller feature, which had been in tech preview, was removed. + +### Version support + +You can now install or upgrade to the following versions: + +* Calico Cloud 21 +* Calico Cloud 20 +* Calico Cloud 19 + +

February 11, 2025 (version 20.4.0)

+ +### New features and enhancements + +#### Exporting custom dashboard cards as CSV files + +You can now export data from your custom dashboard cards as CSV files. +This allows you to analyze the exported data using spreadsheet software or data analysis libraries. + +#### Support for Kubernetes 1.31 + +This release adds support for Kubernetes 1.31. + +#### Control-plane label customization for AKS + +We added support for customizing the namespace labels on AKS clusters. +By default we apply a `control-plane` label to namespaces so that they are exempt from Azure Policy. +If you wish to apply Azure Policy to our namespaces, you can now override this label. + +## January 9, 2025 (web console update) + +### New features and enhancements + +#### Secure web console access with IP allowlists + +We've added support for restricting access to the web console by IP address. +You can now define a list of allowed IP addresses or ranges. +For example, you can allow users to access the web console only from your corporate network. + +To configure an IP allowlist, [open a support ticket](https://tigeraio.my.site.com/community/s/login/). + +

December 19, 2024 (version 20.3.1)

+ +### Bug fixes + +* Previously, updating Calico Cloud introduced breaking changes for some users who had defined network policy tiers with orders higher than the default value. + This was caused by a change to the default tier order to 1,000,000. + Now, during upgrades, the installation script checks to see whether the change in default tier order will cause problems. + If necessary, the script blocks the new installation and prompts the user to adjust their tier order. +* Fixed an issue so that guardian picks up refreshed Service Account tokens which caused an issue on some AKS configurations due to a token expiration change. + +## December 17, 2024 (web console update) + +### New features and enhancements + +#### Dashboards (tech preview) + +We added a set of dashboards to help you understand the activity in your cluster. +Each dashboard is made up of graphs, charts, and diagrams that visually represent the data in your logs. + +You can also create your own arrangement by creating a custom dashboard. +With a custom dashboard, you can combine and arrange cards from any of the other dashboards. + +For more information, see [Dashboards](../observability/dashboards.mdx). + +## December 3, 2024 (version 20.3.0) + +:::info[upgrade notice] +Because of a breaking change, this version is no longer available. +Upgrade instead to Calico Cloud 20.3.1 or later. +::: + +### New features and enhancements + +#### Image Assurance scan result management + +In this release, you can more easily manage your Image Assurance scan results by deleting results you don't need. +On the **All Scan Results** page, select the checkbox next to result item, and then click **Actions > Delete**. +You can also select multiple results and delete them as a bulk action. + +#### Support for Windows nodes on EKS + +We've expanded our support for Windows nodes to include hybrid clusters on Amazon Elastic Kubernetes Service. + +#### Search by policy name or namespace + +$[prodname] now includes search-to-filter capabilities on the policy board and listing pages, which helps you find a specific policy or a subset of policies more quickly. + +#### Envoy deployment as a sidecar + +$[prodname] now provides the ability to deploy Envoy as a sidecar so application layer policy and logging are compatible with other features such as egress gateways, WireGuard for data-in-transit encryption, and Calico’s eBPF data plane. + +For more information, see [Application layer policy](../network-policy/application-layer-policies/alp.mdx) and [L7 logs](../observability/elastic/l7/configure.mdx). + +#### Configurable rules for deep packet inspection + +$[prodname] now provides the ability for administrators to configure and customize the Snort rules that are used in deep packet inspection. +This gives customers greater control over the types of rules that are evaluated. +It also ensures that they can effectively tune and selectively enable rules to phase their deep packet inspection and network-based threat detection. + +For more information, see [Deep packet inspection](../threat/deeppacketinspection.mdx). + +#### Calico early networking (for dual ToR) preserves post-boot default routes + +$[prodname] includes improvements so that early network configuration will be superseded by any BGPPeer or BGPConfiguration resources after successful startup + +For more information see [Deploy a dual ToR cluster](../networking/configuring/dual-tor.mdx). + +#### Enhancements + +* Guardian will respect HTTP proxy environment variables when set on the deployment by mutating webhook configurations. +* Enhanced filtering options in the endpoints page of the web console. +* Performance improvements to Image Assurance processes. + +### Deprecated and removed features + +* All compliance reporting features are deprecated and will be removed in a future release. + +### Updating + +* ***Breaking change:*** Previously, the default tier had no order and was always evaluated last. + Starting with Calico Cloud 20.3.0, the default tier now has an order of 1,000,000. + When upgrading, you must ensure that existing tiers have a lower order or else policy decisions may be affected. + For more information, see [Get started with policy tiers](../network-policy/policy-tiers/tiered-policy.mdx). + + +## November 6, 2024 (version 20.2.0) + +### New features and enhancements + +#### Image Assurance scan result management + +In this release, you can more easily manage your Image Assurance scan results by deleting results you don't need. +On the **All Scan Results** page, select the checkbox next to result item, and then click **Actions > Delete**. +You can also select multiple results and delete them as a bulk action. + +#### Enhancements + +* Various detector improvements, including better handling of historical data and a detector export function. +* Improved webhooks with ability to send global alerts. +* Reduced memory usage for clusters with many `ConfigMap` resources. + +### Bug fixes + +* Fixed an issue with the Image Assurance CLI scanner to ensure it takes CVE exceptions into account when scanning multiple images from the command line at the same time. + +## October 1, 2024 (version 20.1.0) + +### New features and enhancements + +#### View and manage detectors for Container Threat Detection + +We've provided better access to the detectors we use as part of our Container Threat Detection system. +You can now view the complete list of detectors and turn them on or off as you see fit. +Detectors can also be configured as part of a new RuntimeSecurity custom resource. + +#### Create Security Event exceptions for known processes + +We added a way to create Security Event exceptions for processes in your cluster that you know to be safe. +This can be a helpful way to eliminate noise and false positives in your alerts. + +#### Added EPSS data to Image Assurance results + +Image Assurance scans results now include information using the [Exploit Prediction Scoring System (EPSS)](https://www.first.org/epss/). +EPSS scores help you determine the likelihood that a given vulnerability will be exploited in the near future. +Being able to view this information and filter scan results by EPSS score can help you judge the risk of vulnerabilities and prioritize your remediation efforts. + +#### Enhancements + +* Functional and performance improvements to Image Assurance scan results filtering. + +## September 20, 2024 (version 20.0.1) + +### Bug fixes + +* Fixed an issue so that guardian picks up refreshed Service Account tokens which caused an issue on some AKS configurations due to a token expiration change. + +## September 10, 2024 (version 20.0.0) + +### New features and enhancements + +#### Helm customizations + +We've added new options for customizing Helm installations: + +* You can now enable or disable the Compliance and Packet Capture features at installation. +* For many Calico Cloud components, you can specify node selectors, tolerations, and resource requests and limits. + +For more information, see [Connect a cluster to Calico Cloud](../get-started/install-cluster.mdx) and [Install Calico Cloud as part of an automated workflow](../get-started/install-automated.mdx) + +#### Automatic Calico Cloud access for connected IdP groups + +We made it easier for administrators to control access to Calico Cloud by managing users in their existing identity provider groups. +When you add an identity provider, users who have memberships of one or more IdP Groups that have been enabled on Calico Cloud can log in without needing to be invited individually. + +To enable automatic access for your IdP group members, [open a support ticket](https://tigeraio.my.site.com/community/s/login/). + +#### Support for ARM64 + +This release adds support for clusters running on ARM64 architectures. + +#### Support for Kubernetes 1.30 + +This release adds support for Kubernetes 1.30. + +### Deprecated and removed features + +* The honeypods feature has been removed from this release. + +### Version support + +You can now install or upgrade to the following versions: + +* Calico Cloud 20 +* Calico Cloud 19 +* Calico Cloud 18 + +Calico Cloud versions 17 and earlier are no longer supported. + +For information about upgrading, see [Upgrade Calico Cloud](../get-started/upgrade-cluster.mdx). + +## July 23, 2024 (version 19.4.1) + +### Bug fixes + +* For AKS clusters with managed Calico, AKS changed the resources deployed which causes the installer + to fail or for existing clusters the operator will stop managing resources. We made changes to the Calico Cloud + installation to deploy additional resources needed to allow normal operation to continue and installs to proceed. + +## July 9, 2024 (version 19.4.0) + +## New features and enhancements + +### Bulk vulnerability exceptions for Image Assurance + +We added a way to efficiently add large numbers of vulnerability exceptions to your Image Assurance scan results. +Instead of creating exceptions one by one, you can add them all at once by uploading a CSV file with the vulnerability definitions. + +### Bug fixes + +* Previously, defining new values for the crawdad daemon by using the ImageAssurance custom resource had no effect, and the default values remained in place. + This problem is now fixed. +* We fixed a bug that caused problems during Helm upgrades from Calico Cloud versions earlier than 19.1.0. + +## June 11, 2024 (version 19.3.0) + +### New features and enhancements + +#### Jira integration for Image Assurance scan results + +We added a way to create and assign Jira issues directly from your Image Assurance scan results page. +You can filter and prioritize vulnerabilities, and then assign the remediation work to members of your team. +Calico Cloud populates the information you need, including a CSV file with detailed information about the vulnerabilities in your packages. +#### Security events dashboard + +A new dashboard summarizes security events and helps practitioners easily understand how events map across namespaces, MITRE techniques, event types, and attack phases. This allows first responders to quickly make sense of potential threats, engage the right stakeholders, and start the incident response and investigation process. + +For more information, see [Security event management](../threat/security-event-management.mdx). + +#### Exceptions for security events + +$[prodname] now allows users to create exceptions for Security Events with varying levels of scope, from excluding an entire namespace to a specific deployment or workload. This gives operators a way to tune the runtime threat detection they have deployed and focus their investigations and response on critical applications and infrastructure. + +For more information, see [Security event management](../threat/security-event-management.mdx). + +#### New flow logs panel for Endpoints and View Policy pages + +$[prodname] has added new entry points to view flow logs directly from the Endpoints listing and View Policy pages in the UI. +Users can easily see which endpoints are involved in denied traffic, filter on those workloads, and click a link to open a panel that shows associated flows. +A similar link has been added for View Policy pages, which allows users to quickly see the flows that have been recently evaluated by that policy to make sense of denied traffic or updates to rules. + +#### Security Events in Service Graph + +$[prodname] now includes a new tab for Security Events which has taken the Alerts. Most runtime threat detection features now generate Security Events, and their inclusion Service Graph enables users to automatically filter events based on where they are occurring in a cluster. + +#### Security Events IP addresses enriched with ASN and geolocation + +For security events that contain external IP addresses, $[prodname] now automatically performs a geolocation lookup. Understanding the country of origin for an IP address can often be the quickest and easiest way to distinguish legitimate traffic from malicious traffic. + +#### Extend Workload-based WAF to Ingress Gateways + +This latest release enables operators to plug-in a modified, simplified version of WAF to their own instances of Envoy. +This allows users to deploy this version of WAF at the edge of their cluster integrated with an Ingress Gateway (if based on Envoy), with fully customizable rules based on OWASP CoreRuleSet 4.0 and powered by the Coraza engine. + +For more information, see [Deploying WAF with an ingress gateway ](../threat/deploying-waf-ingress-gateway.mdx). + +#### Specifying resource requests and limits in $[prodname] components + +$[prodname] now provides the ability to set resource requests and limits for the components that run as part of $[prodname]. Please see documentation for specific guidance on setting these limits. + +### Known issues + +* It is no longer supported to uninstall $[prodname] on a cluster that before connecting to $[prodname] it had Calico managed with the `addon-manager` component. This includes AKS clusters that had Calico installed and managed by AKS. + +### Bug fixes + +* Fixed an issue where occasionally the status of a cluster would not be updated when the cluster connected resulting in auth tokens and license not being updated in the cluster. + +## April 30, 2024 (version 19.2.0) + +### New features and enhancements + +#### Automated installation with client credentials + +You can now generate and manage client credentials that you can use to automate the Calico Cloud installation process. +With persistent API keys, you can build repeatable installation commands that connect your clusters as part of an automated workflow. + +For more information, see [Install Calico Cloud as part of an automated workflow](../get-started/install-automated.mdx). + +#### Feature options for Helm installations + +For Helm installations, you can now configure some feature options during installation. +You can enable or disable Image Assurance, Container Threat Detection, and the Security Posture Dashboard by adding optional parameters to your Helm command. + +For more information, see [Connect a cluster to Calico Cloud](../get-started/install-cluster.mdx). + +#### Namespace exclusions for image scanning and runtime view + +We added the ability to exclude namespaces from image scanning and runtime view. +By excluding certain namespaces, you can reduce noise in your scan results and focus attention on higher priority workloads. +## April 2, 2024 (version 19.1.0) + +### Bug fixes + +* We fixed a problem that caused the Image Assurance operator to stop working when it reached its memory limit. + +## February 28, 2024 (version 19.0.0) + +### New features and enhancements + +#### Improved flow log filtering for destination domains + +We’ve updated the Felix parameter (`dest_domains`) for DNS policy to make it easy to find only domain names that the deployment connected to (not all the domain names that got translated to the same IP address). +For more information, see [Flow log data types](../observability/elastic/flow/datatypes.mdx). + +#### New flow logs panel on Endpoints page + +We've updated the Endpoints page in the web console with a new flow logs panel so you can view and filter Endpoints associated with denied traffic. Flow log metadata includes the source, destination, ports, protocols, and other key forms. We've also updated the Policy Board to highlight policies with denied traffic. + +#### Improvements to security events dashboard + +We've added the following improvements to the [Security events dashboard](../threat/security-event-management): + +- Jira and Slack webhook integration for security event alerts + + By [configuring security event alerts](../threat/configuring-webhooks), you can push security event alerts to Slack, Jira, or an external HTTP endpoint of your choice. + This lets incident response and security teams to use native tools to respond to security event alerts. + +- Added threat feed alerts + + If you have implemented global threat feeds for suspicious activity (domains or suspicious IPs), alerts are now visible in the Security Overview dashboard. + For more information on threat feeds, see [Trace and block suspicious IPs](../threat/suspicious-ips). + +## Deprecated and removed features + +* The AWS security groups integration is removed in this release. +* The ingress log collection feature is removed in this release. + +## January 31, 2024 (version 18.3.0) + +### New features and enhancements + +#### Assign custom roles to users automatically with Entra ID (formerly Azure AD) groups + +We've added the ability to link custom roles in Calico Cloud to your organization's Entra ID groups. +You can define and modify group membership in Entra ID, and Calico Cloud will automatically grant role-based access to users based on that group membership. + +For more information, see [Create a custom role for an Entra ID group](../users/create-custom-role-for-entra-id-group.mdx). + +#### Export custom roles + +In this release you can export custom roles from a managed cluster and apply them to another managed cluster. +Previously, if you wanted to duplicate roles in multiple clusters, you needed to create them manually for each cluster. +Now there is a process to apply those roles quickly and accurately in all clusters. + +For more information, see [Creating and assigning custom roles](../users/create-and-assign-custom-roles.mdx). + +#### Windows node support for Azure Kubernetes Service + +We've added support for Windows nodes in AKS clusters. + +#### VXLAN support for cluster mesh and federation + +We've expanded our support of cluster mesh to clusters using VXLAN for networking. Cluster mesh can be used to federate services and endpoints to authorize cross-cluster communication with $[prodname] network policies. + +For more information, see [Configure federated endpoint identity and services](../multicluster/kubeconfig.mdx). + +#### Support for Azure CNI with overlay networking for AKS + +We now officially support AKS clusters that are using overlay networking. +This option is useful if you've exhausted your IP addresses. +This option augments existing support for Azure CNI with no overlay (where a VNet IP address is assigned to every pod). + +#### Support for Kubernetes 1.28 + +This release adds support for Kubernetes 1.28. + +### Deprecated and removed features + +* The anomaly detection feature is removed in this release. + If you enabled this feature, you will now stop receiving anomaly detection alerts. +* The AWS security groups integration is deprecated in this release. + It will be removed in a future release. +* The ingress log collection feature is deprecated in this release. + It will be removed in future release. + +### Bug fixes + +* We fixed a problem that stopped diagnostics from being collected after a failed installation. + +## December 21, 2023 (version 18.2.0) + +### New features and enhancements + +#### Security Posture Overview dashboard + +We've added a new Security Posture Overview dashboard that helps you assess the security posture of your cluster. +Using a list of prioritized recommended actions, you can start to take steps to reduce your risk over time. +Because the dashboard is based on existing Calico Cloud data, no configuration is required. +You can start improving the security posture of your Kubernetes cluster immediately. + +#### Support for RKE2 + +This release comes with support for connecting RKE2 clusters to Calico Cloud. + +### Known issues + +* You can't connect RKE2 clusters to Calico Cloud if you enabled the Image Assurance runtime view feature on any of your managed clusters. +As a workaround, disable runtime view before connecting your RKE2 cluster. + +### Bug fixes + +* Code changes to the Kubernetes controller-runtime led to intermittent errors in how the Container Threat Detection status was displayed in the web console. +We modified the Runtime Security operator to account for these changes. + +## November 29, 2023 (version 18.1.0) + +### New features and enhancements + +* We limited the permissions that are assigned to the Calico Cloud installer. +Previously, the installer had cluster administrator privileges. +Now the installer gets access only to what is required to install Calico Cloud. +* **Image Assurance**. We added a filter that lets you sort your list of running images by severity rating. + +### Known issues + +* If you update your cluster to a previous version of Calico Cloud (18.0.0 or earlier), you may see an erroneous message about a failed installation that took place before the successful installation. +This failed installation message can be disregarded. + +### Bug fixes + +* Fixed an issue that caused security events generated by AKS managed clusters to be missing pod and namespace information. +* Fixed an issue that caused some pods on managed clusters to crash-loop after upgrading clusters that also have Dynatrace running. + +## October 23, 2023 (version 18.0.0) + +### New features and enhancements + +#### Image Assurance registry scanner + +The Image Assurance feature adds the ability to scan images in container registries at any time, on any infrastructure, including Kubernetes. This is ideal protection for images that don’t go through a pipeline (for example, third-party images), but are published to a registry. If CVEs are missed in your build pipeline, you can catch them before they are deployed. +#### Security event management + +We've added a new Security event management dashboard for threat detection. +Security events provides context for suspicious activity detected in your cluster. +Combined with the Kubernetes context, you can see what workloads are affected. + +For more information, see [Security event management](../threat/security-event-management.mdx). + +#### New performance optimizations for egress gateways + +$[prodname] includes new performance options for egress gateway policies that can be used to ensure that application client and gateway pods are on the same cluster node. + +For more information, see [Optimize egress networking for workloads with long-lived TCP connections](../networking/egress/egress-gateway-maintenance.mdx). + +#### Configurable XFF headers for Envoy + +We've added support for XFF to propagate the original IP address when proxying application layer traffic with Envoy within a Kubernetes cluster. + +For more information, see [Tigera Operator API reference](../reference/installation/api.mdx#envoysettings). + +#### Alert-only mode for workload-based Web Application Firewall (WAF) + +We've added a new default mode for WAF that is monitor/event only. +This allows operators and security teams to verify the accuracy of configured rules before actively blocking traffic. + +For more information, see [Web application firewall](../threat/web-application-firewall.mdx). + +#### Enhancements + +* You can now remove disconnected clusters from the list of managed clusters. See [Cluster management](../operations/cluster-management.mdx#remove-a-cluster). + +### Known issues + +{/* EV-4175 */} +* The policy recommendation tool is not displaying policy recommendations. +There is currently no workaround, but the issue will be fixed in an upcoming release. + +* Using a bookmarked link to log in to the $[prodname] UI in this release causes the following problems: +- Image Assurance configuration page fails to load +- the web console shows the enterprise license countdown at the top of the screen even for paid customers + +If you are experiencing any login issues, go to https://calicocloud.io and log back in. + +* Calico panics if kube-proxy or other components are using native `nftables` rules instead of the `iptables-nft` compatibility shim. Until Calico supports native nftables mode, we recommend that you continue to use the iptables-nft compatibility layer for all components. (The compatibility layer was the only option before Kubernetes v1.29 added alpha-level `nftables` support.) Do not run Calico in "legacy" iptables mode on a system that is also using `nftables`. Although this combination does not panic or fail (at least on kernels that support both), the interaction between `iptables` "legacy" mode and `nftables` is confusing: both `iptables` and `nftables` rules can be executed on the same packet, leading to policy verdicts being "overturned". Note that this issue applies to all previous versions of $[prodname]. + +### Bug fixes + +{/* TSLA-5314 */} +* We fixed an issue that caused a bad error message to appear when you changed a Calico Cloud user's role. + +## September 11, 2023 (version 17.1.1) + +### New features and enhancements + +* We redesigned a section of the Image Assurance UI to make it easier to see how vulnerable an image is. +* We made improvements to the way Container Threat Detection processes large volumes of alerts. + +### Bug fixes + +* We fixed a problem that caused the user interface to crash when a user attempted to edit a policy. +* We fixed a problem that prevented certain images from being scanned. + +## September 5, 2023 (version 17.1.0) + +### New features and enhancements + +#### Improvements to software versioning for Calico Cloud installations on managed clusters + +We've made it easier to see what version of Calico Cloud you're running or installing on a managed cluster. +Now you can: +* view the Calico Cloud version number for each connected cluster from the Managed Clusters page +* see when an update is available for a managed cluster +* select a specific Calico Cloud version to install when you connect a cluster + +#### Security Events UI page + +Alerts corresponding to detections generated by container threat detection will now be published to the Security Events UI page, found within the Threat Defense left navigation menu item. + +On this page, for every security event detected, users can view: + +* security event name +* security event type +* severity level +* a description of what suspicious activity has been detected +* impacted assets (pod and namespace) +* attack vector type +* Mitre tactic and techniques associated with the detection +* mitigation recommendation +* additional metadata and context associated with the detection. + +Alerts will continue to also appear on the Alerts UI page. + +For more information, see [Security event management](../threat/security-event-management.mdx). + +### Bug fixes + +* Runtime Security alerts now correctly show the generated_time as the time the alert was generated. Previously they incorrectly showed the time when the underlying event which caused the alert was generated. +* Runtime Security alerts now correctly show the associated Mitre information. + +### Known issues + +* Enabling WAF and Container Threat Detection through the UI is not possible for clusters running Kubernetes v1.27+. Both features can be enabled using kubectl. +* If you connected your cluster to Calico Cloud using Helm before the release of version 17.1.0, reinstalling or upgrading to any version of Calico Cloud may result in an error: "Error: rendered manifests contain a resource that already exists." +Previously, the `installers.operator.calicocloud.io` custom resource definition (CRD) installed by Helm required manual upgrades. +After the release of Calico Cloud 17.1.0, this CRD is updated automatically, but this change causes errors the first time you attempt to reinstall or upgrade Calico Cloud on a cluster that was connected using Helm before the release of Calico Cloud 17.1.0. + +As a workaround, label the CRD so that it is managed by Helm by running the following command: + +```bash +kubectl patch crd installers.operator.calicocloud.io -p '{"metadata": {"annotations": {"meta.helm.sh/release-name":"calico-cloud-crds","meta.helm.sh/release-namespace":"calico-cloud"},"labels":{"app.kubernetes.io/managed-by":"Helm"}}}' +``` + +This allows you to successfully reinstall or upgrade to Calico Cloud by following the procedure in [Upgrade Calico Cloud](../get-started/upgrade-cluster.mdx). + +### Security updates + +* Runtime security upgraded to [golang 1.20.7](https://go.dev/doc/devel/release#go1.20.7), which includes security updates. +* We rebuilt `cc-operator` and `cc-cni-config-scanner`, which has reduced the number of CVEs. + +## August 21, 2023 (version 17.0.0) + +### New features and enhancements + +#### New policy recommendations engine for namespace isolation + +$[prodname] has added a new policy recommendations engine that automatically generates staged policies for namespace isolation within your cluster. [Policy recommendations](../network-policy/recommendations/policy-recommendations). + +#### Destination-based routing for egress gateways + +$[prodname] introduces a new mode for egress gateways that can leverage destination-based routing. Destination-based routing for egress gateways allows operators associated with a destination that is external to a Kubernetes cluster (for example, IP address or CIDR), to a specific egress gateway deployment. [Egress gateways](../networking/egress/egress-gateway-on-prem). + +#### Support for DNS rules in clusters using NodeLocal DNSCache + +$[prodname] has added support for DNS rules in clusters using NodeLocal DNSCache. Also related, there is new documentation on using Calico policy to secure DNS traffic within the cluster with NodeLocal DNSCache enabled. [Use NodeLocal DNSCache in your cluster](../networking/configuring/node-local-dns-cache). + +#### Improved UI for configuring Workload-based Web Application Firewall (WAF) + +$[prodname] includes updates to the UI that allows you to select which services are enabled for the Workload-based Web Application Firewall. [Web application firewall](../threat/web-application-firewall). + +#### WireGuard support for AKS and EKS with Calico CNI + +$[prodname] now offers official support for WireGuard when using Microsoft AKS or Amazon EKS with Calico CNI. This mode of deployment offers performance benefits and a more efficient routing table compared to using cloud provider CNIs. [Encrypt data in transit](../compliance/encrypt-cluster-pod-traffic). + +#### Additional custom roles for $[prodname] + +You can now create custom role-based access controls for two new roles: "Usage Metrics" and "Image Assurance Admin". + +#### Image Assurance improvements + +* A containerized version of Image Assurance scanner is now available to integrate into your CI/CD platform. +See [Image Assurance containerized scanner](https://quay.io/repository/tigera/image-assurance-scanner-cli) to pull the latest image. +* Substantial UI improvements including a new package-centric view of images + +### Known issues + +* The canvas on Service Graph may zoom and pan unexpectedly when modifying Views or Layers +* Dragging tiers to modify their order is currently not working in the UI, though you can still change its order when editing a tier +* Policy recommendations may generate rules with ports and protocols for intra-namespace traffic. +This will be modified in the next patch release to exclude ports and protocols and provide an option to Allow or Pass this traffic. + +## June 6, 2023 + +### New features and enhancements + +#### In-cluster scanning with Image Assurance + +Calico Cloud now includes the ability to scan and monitor the images running in your Kubernetes clusters for new vulnerabilities. +In-cluster scanning will scan any new images not previously scanned, and continuously monitor the BOM (Bill of Materials) for running images that have prior scan results. + +#### New detectors for container threat detection + +We've added several new detectors for container threat detection. +These detectors help identify unsanctioned use of network tools, task scheduling, container admin and Docker commands, and much more. +Calico Cloud now includes over 40 different detectors across each category of the [MITRE ATT&CK Matrix](https://attack.mitre.org/). + +## May 2, 2023 + +This release includes a number of performance improvements and bug fixes. + +## April 24, 2023 + +### Depreciated support for RKE, RKE2 + +Calico Cloud no longer supports installation on RKE or RKE2. + +## April 11, 2023 + +## New features and enhancements + +### Updates to Managed Clusters + +The **Managed Clusters** page has been redesigned to make it easier and more intuitive to search and filter your clusters. + +#### Egress gateways for AKS and Azure + +$[prodname] adds egress gateway support for Microsoft Azure and AKS. Egress gateways allow you to identify the namespaces associated with egress traffic outside of your cluster. [Egress gateways for AKS and Azure](../networking/egress/egress-gateway-azure). + +#### UI for workload-based Web Application Firewall (WAF) + +$[prodname] includes a new UI to enable and configure a workload-based Web Application Firewall. For more information, see [Workload-based web application firewall](../threat/web-application-firewall.mdx#enable-waf). + +#### Application layer policy with Envoy + +$[prodname] now includes support for application layer policy with Envoy, enabling platform operators to define authorization rules in $[prodname] policies for protocols such as HTTP and gRPC. For more information, see [Application layer policies](../network-policy/application-layer-policies/). + +#### Service Graph performance optimizations + +$[prodname] added several optimizations to improve the performance of Service Graph for clusters with larger numbers of namespaces. + +#### Improvements to Envoy to accommodate advanced ingress controllers + +$[prodname] improves its Envoy deployment so you can use this feature in clusters with ingress controllers that perform advanced load balancing. For more information, see [Workload-based web application firewall](../threat/web-application-firewall.mdx). + +#### Improved $[prodname] component security + +$[prodname] components were updated with more restrictive access for pods and containers using the Kubernetes security context: +* Non-root context whenever possible +* Root context and privilege escalation are used only when necessary +* Added `drop ALL capabilities` for pod security +* Enabled `RuntimeDefault` as the default seccomp profile for all workloads + +## February 28, 2023 + +### New features and enhancements + +* Adds Bottlerocket support for Container Threat Detection. +* Adds support for scanning multiple images with Image Assurance + +### Bug fixes + +* Fixes "Kibana" menu item rename to "Logs". +* Bug fixes for Container Threat Detection alerts. + +## February 7, 2023 + +### New features and enhancements + +#### New and improved Dashboards + +Calico Cloud includes new and improved Dashboards that enable operators to define cluster* and namespace-scoped dashboards with new modules for policy usage, application layer and DNS metrics, and much more. + +#### Configure Threat Feeds in the Calico Cloud UI + +Calico Cloud includes a new UI that can be used to manage and configure global threat feeds + +For more information, see [Trace and block suspicious IPs](../threat/suspicious-ips.mdx). + +#### Namespace-based policy recommendations + +Calico Cloud has improved its policy recommendation engine to add namespace-based recommendations. +This enables operators to easily implement microsegmentation for namespaces. + +For more information, see [Create policy recommendation](../network-policy/recommendations/policy-recommendations.mdx). + +#### Create custom roles for Calico Cloud users + +Calico Cloud administrators can now define granular roles and permissions for users using custom role-based access controls. + +For more information, see [Create and assign custom roles](../users/create-and-assign-custom-roles.mdx). + +#### Egress gateway improvements + +Calico Cloud has improved the probes to check readiness and outbound connectivity of egress gateways. +Calico Cloud has also rearchitected egress gateway pods to improve security and make use of a temporary init container to set up packet forwarding. + +#### Image Assurance updates + +* CLI Version v1.3.4 +* Calico Cloud supports the Image Assurance CLI scanner versions 1.3.0 and later. +* **Bug fix:** Previously, the scanner returned an error if it reached a size limit while uploading vulnerabilities. +This size limit has been removed. + +## December 13, 2022 + +### New features and enhancements + +#### Search by CVE in Image Assurance + +Image Assurance reporting features now includes a search and filtering capability that allows you to find list items based on a single CVE ID within any Image Assurance reports. + +#### Enable and disable Container Threat Detection in the Calico Cloud UI + +You can now enable or disable Container Threat Detection within the UI. After enabling the feature, you can review the status of which nodes are being monitored by the feature and which nodes of your cluster are unsupported. + +#### New Feature: Calico Cloud Service Status Page + +All users can view the status and health of the Calico Cloud service on our new status page: [https://status.calicocloud.io](https://status.calicocloud.io/). + +## November 1, 2022 + +### Image Assurance + +CLI Version v1.1.2. + +New CLI will now check that it is compatible with the latest Image Assurance API. + +### Container Threat Detection + +![tech-preview](/img/calico-cloud/tech-preview.svg) + +Release of Container Threat Detection + +With Container Threat Detection, you can monitor container activity using eBPF. Enable this feature to receive alerts based on file and process activity for known malicious and suspicious behavior. Alert events can be viewed on the Alerts page in the web console. +## September 26, 2022 + +### New feature: Helm + +$[prodname] now supports [installation using Helm](../get-started/install-cluster.mdx). + +### New feature: Private Registry + +$[prodname] now supports [installation from private registries](https://docs.calicocloud.io/get-started/connect/install-cluster). Note that this is only supported when installing with Helm. + +### Expanded platform support: RKEv2 + +Installation works on clusters with Calico deployed by RKEv2. + +## September 12, 2022 + +### Image Assurance is GA + +Image Assurance is now released for general availability. + +With Image Assurance, DevOps and platform teams can scan images in public and private registries, including images that are automatically discovered in connected clusters. +Image Assurance provides a runtime view into risk, based on known vulnerabilities. +It also offers admission controller policies to block resources in Kubernetes from creating containers with vulnerable images from entering your cluster. + +#### Changes from the tech preview version + +**New Image Assurance CLI scanner** +Image scanning is now configured and performed by the `tigera-scanner` CLI. +You can integrate `tigera-scanner` into your CI/CD pipelines to ensure builds are checked by Image Assurance before deployment. +You can also use the CLI scanner offline and on-demand for ad hoc scanning and emergency patching. + +**Export options for vulnerability scan results and runtime views** + +We've made it easier for platform operators to share Image Assurance scan results and runtime views with these export options: + +* Export one row per image or one row per image and CVE. +* Export CSV or JSON files. + +### Malware detection is GA + +Malware detection is now released for general availability. + +Calico Cloud's malware detection identifies malicious files in your cluster and generates alerts. +Calico Cloud uses eBPF-based monitoring to log file hashes of programs running in your cluster. +If there's a match to known malware from our threat intelligence library, you receive an alert. +You can view your alerts on the _Alerts_ page on the web console. +## July 27, 2022 + +Improvement: Export logs to a SIEM + +To help meet your compliance requirements, we've added documentation to export logs to a SIEM (syslog, Splunk, or Amazon S3). See [Export logs to a SIEM](../observability/elastic/archive-storage.mdx). + +## July 7, 2022 + +### New feature: Distributed Web Application Firewall (WAF) with Envoy + +![tech-preview](/img/calico-cloud/tech-preview.svg) + +$[prodname] now includes the option to enable Web Application Firewall (WAF) rule sets when using Envoy as a daemonset. This enables operators to implement an additional layer of security and threat detection for application layer traffic. See [Workload-based Web Application Firewall (WAF)](../threat/web-application-firewall.mdx). + +### New Feature: Configuration option to use DNS rules with StagedNetworkPolicies + +$[prodname] has added a new configuration option in Felix (`DNSPolicyMode`) that lets you audit DNS rules with StagedNetworkPolicies. There is a small performance trade off if you enable this option, so we recommended to disabling it when it’s not required. See [Felix configuration](../reference/resources/felixconfig.mdx#dnspolicymode). + +### Improvement: Additional predefined RBAC options + +$[prodname] now supports 3 more pre-defined RBAC controls (DevOps, security and compliance persona) for role assignment. + +### Improvement: Anomaly detection deployment + +![tech-preview](/img/calico-cloud/tech-preview.svg) + +$[prodname] has made the configuration and deployment of anomaly detection jobs for threat detection and performance hotspots more granular, allowing you to selectively enable jobs depending on your use case. + +### Improvement: the web console now displays cluster installation progress and streaming logs + +$[prodname] now displays information about managed cluster install progress right in the UI. + +After you run the install command (**Connect Cluster** wizard in Managed Clusters), installation progress is automatically displayed along with logs for the managed cluster. + +## May 10, 2022 + +### New feature: Visibility into usage metrics + +$[prodname] now displays information about cloud usage metrics. This will provide visibility into the node hours and data ingested for consumption-based invoices. + +Account owners can click the new "Usage Metrics" button at the bottom of the left navbar to navigate to the new page. + +### Expanded platform support: AKS with managed Calico + +Installation works on clusters with Calico deployed by AKS. + +## April 26, 2022 + +### New feature: Malware detection + +![tech-preview](/img/calico-cloud/tech-preview.svg) + +$[prodname] introduces malware detection in tech preview, which uses eBPF-based monitoring to log observed file hashes of programs running in your $[prodname] Kubernetes clusters. Malware detection identifies malicious files by comparing observed file hashes with our threat intelligence library of known malware, and generates alerts when malware is detected in your cluster. Alerts can be viewed on the Alerts page of the web console. + +**If you started using $[prodname] before January 24, 2022**, you must upgrade your existing cluster to get malware detection: + +1. Navigate to the **Managed Clusters** page. +1. Select the cluster from the list, and click **Reinstall**. +1. Copy the updated install script command and run it against your cluster. + +## April 20, 2022 + +### Improved installation + +We’ve updated the $[prodname] installation process to improve security, reduce dependencies on utilities (such as bash), and allow you to customize the name of your connected clusters. + +The $[prodname] installation process will now require running a `kubectl apply` command instead of a bash script. Additionally, the installation script has been moved behind an authenticated endpoint. The updated install script is now available on the **Managed Clusters** page of the $[prodname] UI. + +**If you started using $[prodname] before January 24, 2022**, you must upgrade your existing cluster to get these changes: + +1. Navigate to the **Managed Clusters** page. +1. Select the cluster from the list, and click **Reinstall**. +1. Copy the updated install script command and run it against your cluster. + +## April 19, 2022 + +### New feature: Image Assurance + +![tech-preview](/img/calico-cloud/tech-preview.svg) + +$[prodname] introduces Image Assurance in tech preview, enabling DevOps and platform teams to scan images in public and private registries, and images that are automatically discovered in connected clusters. Image Assurance provides a runtime view into risk, based on discovered vulnerabilities. It also offers admission controller policies to enforce how vulnerable images are used to create resources within Kubernetes. diff --git a/calico-cloud_versioned_docs/version-24-1/releases.json b/calico-cloud_versioned_docs/version-24-1/releases.json new file mode 100644 index 0000000000..2319b0d3ae --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/releases.json @@ -0,0 +1,178 @@ +[ + { + "title": "v3.24.0-1.0", + "tigera-operator": { + "version": "v1.43.0", + "image": "tigera/operator", + "registry": "quay.io" + }, + "calico": { + "minor_version": "v3.32", + "archive_path": "archive" + }, + "components": { + "alertmanager": { + "version": "v3.24.0-1.0", + "image": "tigera/alertmanager" + }, + "calico": { + "version": "v3.24.0-1.0", + "image": "tigera/calico" + }, + "calicoq": { + "version": "v3.24.0-1.0", + "image": "tigera/calicoq" + }, + "compliance-benchmarker": { + "version": "v3.24.0-1.0", + "image": "tigera/compliance-benchmarker" + }, + "coreos-alertmanager": { + "version": "v0.33.0" + }, + "coreos-config-reloader": { + "version": "v0.91.0" + }, + "coreos-dex": { + "version": "v2.45.1" + }, + "coreos-fluentd": { + "version": "1.19.3" + }, + "coreos-prometheus": { + "version": "v3.12.0" + }, + "coreos-prometheus-operator": { + "version": "v0.91.0" + }, + "deep-packet-inspection": { + "version": "v3.24.0-1.0", + "image": "tigera/deep-packet-inspection" + }, + "dex": { + "version": "v3.24.0-1.0", + "image": "tigera/dex" + }, + "dikastes": { + "version": "v3.24.0-1.0", + "image": "tigera/dikastes" + }, + "eck-elasticsearch": { + "version": "8.19.19" + }, + "eck-elasticsearch-operator": { + "version": "3.4.1" + }, + "eck-kibana": { + "version": "8.19.19" + }, + "egress-gateway": { + "version": "v3.24.0-1.0", + "image": "tigera/egress-gateway" + }, + "elastic-tsee-installer": { + "version": "v3.24.0-1.0", + "image": "intrusion-detection-job-installer" + }, + "elasticsearch": { + "version": "v3.24.0-1.0", + "image": "tigera/elasticsearch" + }, + "elasticsearch-operator": { + "version": "v3.24.0-1.0", + "image": "eck-operator" + }, + "envoy": { + "version": "v3.24.0-1.0", + "image": "tigera/envoy" + }, + "firewall-integration": { + "version": "v3.24.0-1.0", + "image": "tigera/firewall-integration" + }, + "fluentd": { + "version": "v3.24.0-1.0", + "image": "tigera/fluentd" + }, + "fluentd-windows": { + "version": "v3.24.0-1.0", + "image": "tigera/fluentd-windows" + }, + "gateway-api-envoy-gateway": { + "version": "v3.24.0-1.0", + "image": "envoy-gateway" + }, + "gateway-api-envoy-proxy": { + "version": "v3.24.0-1.0", + "image": "envoy-proxy" + }, + "gateway-api-envoy-ratelimit": { + "version": "v3.24.0-1.0", + "image": "envoy-ratelimit" + }, + "gateway-l7-collector": { + "version": "v3.24.0-1.0", + "image": "tigera/gateway-l7-collector" + }, + "intrusion-detection-controller": { + "version": "v3.24.0-1.0", + "image": "tigera/intrusion-detection-controller" + }, + "istio-install-cni": { + "version": "v3.24.0-1.0", + "image": "tigera/istio-install-cni" + }, + "istio-pilot": { + "version": "v3.24.0-1.0", + "image": "tigera/istio-pilot" + }, + "istio-proxyv2": { + "version": "v3.24.0-1.0", + "image": "tigera/istio-proxyv2" + }, + "istio-ztunnel": { + "version": "v3.24.0-1.0", + "image": "tigera/istio-ztunnel" + }, + "kibana": { + "version": "v3.24.0-1.0", + "image": "tigera/kibana" + }, + "manager": { + "version": "v3.24.0-1.0", + "image": "tigera/manager" + }, + "node": { + "version": "v3.24.0-1.0", + "image": "tigera/node" + }, + "node-windows": { + "version": "v3.24.0-1.0", + "image": "tigera/node-windows" + }, + "prometheus": { + "version": "v3.24.0-1.0", + "image": "tigera/prometheus" + }, + "prometheus-config-reloader": { + "version": "v3.24.0-1.0", + "image": "tigera/prometheus-config-reloader" + }, + "prometheus-operator": { + "version": "v3.24.0-1.0", + "image": "tigera/prometheus-operator" + }, + "tigera-cni-windows": { + "version": "v3.24.0-1.0", + "image": "cni-windows" + }, + "tigera-third-party-cni-plugins": { + "version": "v3.24.0-1.0", + "image": "third-party-cni-plugins" + }, + "upstream-istio": { + "version": "1.29.2" + } + } + } +] diff --git a/calico-cloud_versioned_docs/version-24-1/threat/configuring-webhooks.mdx b/calico-cloud_versioned_docs/version-24-1/threat/configuring-webhooks.mdx new file mode 100644 index 0000000000..3f0cf16d61 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/configuring-webhooks.mdx @@ -0,0 +1,48 @@ +--- +description: Configure Calico Cloud webhooks from the web console to post security event alerts to Slack, Jira, Alertmanager, or generic JSON endpoints. +title: Webhooks for security events +--- + +# Webhooks for security event alerts + +You can configure $[prodname] webhooks to post security alerts directly to Slack, Jira, Alertmanager, or any custom HTTP endpoint. + +## Before you begin + +Your target application must be configured to receive data from the $[prodname] webhook. + +* **Slack.** You must have a webhook URL for the Slack app that you want $[prodname] to send alerts to. + See [Sending messages using Incoming Webhooks](https://api.slack.com/messaging/webhooks) for more information. +* **Jira.** You must have an API token for an Atlassian user account that has write permissions to your Jira instance. + See [Manage API tokens for your Atlassian account](https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/) for details on how to obtain an API token. + You also need: + * Your Atlassian site URL. If you access Jira at the URL `https://.atlassian.net/jira`, then your site URL is `.atlassian.net`. + * A Jira project key. This is the Jira project where your $[prodname] webhook creates new issues. This user associated with your API token must have write permissions to this project. +* **Alertmanager.** You must have a webhook URL for your Alertmanager instance to send alerts to. + The webhook URL for an Alertmanager service accessible from within your cluster is: + + ```bash + http://..svc.cluster.local:9093/api/v2/alerts + ``` + + Replace the following: + * ``: the name of your Alertmanager service. + * ``: the namespace associated with your Alertmanager service. + If no namespace is specified in the Alertmanager service resource, then this value is `default`. + + Alertmanager integration in $[prodname] leverages the API v2 exposed by the Alertmanager. + For more information, see the [Alertmanager API documentation](https://prometheus.io/docs/alerting/latest/alerts_api/). +* **Generic JSON.** You must have a webhook URL for any other application you want the $[prodname] webhook to send alerts to. + +## Create a webhook for security event alerts + +1. In the web console, select **Activity** > **Webhooks**, and then click **Create your first webhook**. +2. Enter a **Name** for your webhook, select which **Event types** you want to get alerts for, and, under **Type**, select whether to configure the webhook for Slack, Jira, Alertmanager, or for generic JSON output. +3. Complete the fields for your webhook type and click **Create Webhook**. +4. To check that your webhook is working, find the webhook on the **Webhooks** page and click **Actions > Test Webhook**. + If your webhook is set up correctly, you should generate a HTTP response. + + ```bash title="Example: Successful HTTP response" + 200 OK + Webhook sent successfully + ``` \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/threat/deeppacketinspection.mdx b/calico-cloud_versioned_docs/version-24-1/threat/deeppacketinspection.mdx new file mode 100644 index 0000000000..7a84feb420 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/deeppacketinspection.mdx @@ -0,0 +1,188 @@ +--- +description: Run deep packet inspection on selected workloads in Calico Cloud connected clusters with Snort community rules to alert on suspected malicious traffic. +--- + +# Deep packet inspection + +## Big picture + +Configure deep packet inspection (DPI) in clusters to get alerts on compromised resources. + +## Value + +Security teams need to run DPI quickly in response to unusual network traffic in clusters so they can identify potential threats. +Also, it is critical to run DPI on select workloads (not all) to efficiently make use of cluster resources and minimize the impact of false positives. +$[prodname] provides an easy way to perform DPI using [Snort community rules](https://www.snort.org/downloads/#rule-downloads). +You can disable DPI at any time, selectively configure for namespaces and endpoints, and alerts are generated in the Alerts dashboard in the web console. + +## Concepts + +For each deep packet inspection resource (DeepPacketInspection), $[prodname] creates a live network monitor that inspects the header and payload information of packets that match the Snort community rules. +Whenever malicious activities are suspected, an alert is automatically added to the Alerts page in the $[prodname] web console. + +$[prodname] DPI uses AF_PACKET, a Linux socket that allows an application to receive and send raw packets. +It is commonly used for troubleshooting (like tcpdump and Wireshark), but also for network intrusion detection. For details, see [AF_Packet](https://man7.org/linux/man-pages/man7/packet.7.html). + +## Before you begin + +**Not supported:** + +- Multi-nic setup +- $[prodname] nodes running Windows hosts + +## How to + +- [Configure deep packet inspection](#configure-deep-packet-inspection) +- [Configure resource requirements](#configure-resource-requirements) +- [Access alerts](#access-alerts) +- [Verify deep packet inspection is running](#verify-deep-packet-inspection-is-running) + +### Configure deep packet inspection + +Create a YAML file containing one or more [DeepPacketInspection](../reference/resources/deeppacketinspection.mdx) resources and apply it to your cluster. + +```bash +kubectl apply -f +``` + +To stop deep packet inspection, delete the DeepPacketInspection resource from your cluster. + +```bash +kubectl delete -f +``` + +**Examples of selecting workloads** + +Following is a basic example that selects a single workload that has the label `k8s-app` with the value `nginx`. + +```yaml +apiVersion: projectcalico.org/v3 +kind: DeepPacketInspection +metadata: + name: sample-dpi-nginx + namespace: sample +spec: + selector: k8s-app == "nginx" +``` + +In the following example, we select all workload endpoints in the `sample` namespace. + +```yaml +apiVersion: projectcalico.org/v3 +kind: DeepPacketInspection +metadata: + name: sample-dpi-all + namespace: sample +spec: + selector: all() +``` + +### Configure resource requirements + +Adjust the CPU and RAM used for performing deep packet inspection by updating the [component resource in IntrusionDetection](../reference/installation/api.mdx#intrusiondetectioncomponentresource). + +For a data transfer rate of 1GB/sec on workload endpoints being monitored, we recommend a minimum of 1 CPU and 1GB RAM. + +The following example configures deep packet inspection to use a maximum of 1 CPU and 1GB RAM. + +```yaml +apiVersion: operator.tigera.io/v1 +kind: IntrusionDetection +metadata: + name: tigera-secure +spec: + componentResources: + - componentName: DeepPacketInspection + resourceRequirements: + limits: + cpu: '1' + memory: 1Gi + requests: + cpu: 100m + memory: 100Mi +``` + +### Access alerts + +The alerts generated by deep packet inspection are available in the web console in the Alerts page. + +### Verify deep packet inspection is running + +Get the [status of DeepPacketInspection](../reference/resources/deeppacketinspection.mdx#status) resource to verify if live traffic is being monitored on selected workload endpoints. + +```bash +kubectl get -n +``` + +### Install custom Snort rules + +If you don't want to use the [Snort community rules](https://www.snort.org/downloads/#rule-downloads), you can install a custom set of Snort rules to perform deep packet inspections. +You may want to install your own rules if: +* you use a paid subscription to a Snort ruleset +* you have written your own Snort rules. + +Beyond custom rules, if you have a paid Snort subscription, you can also customize Snort configuration to fine-tune how DPI generates alerts. For example, you can: +* **Limit alert rates** to reduce the volume of alerts generated for noisy rules +* **Suppress alerts** for specific rules or traffic sources that are known to be benign +* **Apply rate filters** to dynamically change alert behavior based on traffic patterns + +These customizations are managed through the Snort configuration file (`snort.lua`) and are mounted into the DPI container using the same initContainers mechanism described below. For details on configuring filters, suppressions, and rate limits, see the [Snort3 documentation](https://docs.snort.org/). + +:::important +If you install custom Snort rules, $[prodname] will stop updating the community rules with each minor release. +You will be responsible for making sure your rules are up to date. +::: + +1. Create a container with your custom Snort rules. + + 1. Copy your Snort rule files into a new `./snort-rules/` directory. + 1. Create a `Dockerfile` like this one: + + ```yaml + FROM alpine:3.14 + COPY snort-rules /snort-rules + ENTRYPOINT [ "/bin/sh", "-c", "cp /snort-rules/* /usr/etc/snort/rules/" ] + ``` + 1. In the console, run the following commands: + + ```bash + docker build . -t : + docker push : + ``` + + After the image has been pushed to the registry, you're ready to configure the `IntrusionDetection` resource. + +1. Update the `IntrusionDetection` resource with the custom Snort rules image. + + ```yaml + spec: + deepPacketInspectionDaemonset: + spec: + template: + spec: + initContainers: + - name: snort-rules + image: : + ``` + + This can also be done by running the following command: + + ```bash + kubectl patch intrusiondetection tigera-secure --type merge -p '{"spec":{"deepPacketInspectionDaemonset":{"spec":{"template":{"spec":{"initContainers":[{"name":"snort-rules", "image":":"}]}}}}}}' + ``` + +1. Verify that your custom rules have been installed correctly: + + 1. If it's not running already, [apply the DeepPacketInspection resource](#configure-deep-packet-inspection) to your cluster. + 1. Extract the list of Snort rules that are currently being used by running the following commands: + + ```bash + export POD=$(kubectl get pods -n tigera-dpi -o custom-columns=:metadata.name --no-headers | head -n 1) \ + kubectl exec -n tigera-dpi $POD -- tar -cf - /usr/etc/snort/rules | tar --strip-components=4 -xf - + ``` + + 1. If these rules match those in your custom set, then the installation was successful. + +## Additional resources + +- [Configure packet capture](../observability/packetcapture.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/threat/deploying-waf-ingress-gateway.mdx b/calico-cloud_versioned_docs/version-24-1/threat/deploying-waf-ingress-gateway.mdx new file mode 100644 index 0000000000..53ba453c1f --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/deploying-waf-ingress-gateway.mdx @@ -0,0 +1,142 @@ +--- +description: Step-by-step tutorial for deploying a Calico Cloud web application firewall with the Calico Ingress Gateway to protect publicly exposed services from Layer 7 attacks. +--- + +# Deploy a web application firewall with Calico Ingress Gateway + +This guide shows you how to deploy and configure a web application firewall (WAF) with the Calico Ingress Gateway. +By deploying the WAF in this way, you can protect publicly exposed services from a variety of application-layer attacks. + +:::note + +This feature is tech preview. Tech preview features may be subject to significant changes before they become GA. + +::: + +:::caution + +This page describes the original, Gateway-wide WAF for Calico Ingress Gateway. It is superseded by [Gateway WAF](./gateway-waf/overview.mdx), which adds per-route and per-namespace policies, custom rules, and compliance validation. For Gateway-wide protection equivalent to this guide, attach a `WAFPolicy` to your `Gateway` — see [Attach WAF to a route](./gateway-waf/attach-waf-to-a-route.mdx). This page will be removed in a future release. + +::: + +## Prerequisites + +* [Calico Ingress Gateway](../networking/ingress-gateway/about-calico-ingress-gateway.mdx) is set up for your cluster. +* You have the value of `metadata.name` for the `Gateway` resource you want to use with WAF. + +## Deploy WAF on Calico Ingress Gateway + +Calico Ingress Gateway uses Envoy's implementation of the Kubernetes Gateway API. +To direct ingress gateway traffic through the WAF, you need to configure Envoy to use the WAF HTTP filter. + +To enable WAF on a Calico Ingress Gateway: + +1. Create a `Backend` resource to allow Envoy to find the WAF HTTP filter: + + ```bash + kubectl apply -f - < + EOF + ``` + + Replace `` with the value of `metadata.name` in your `Gateway` resource. + + The WAF now processes all HTTP requests that pass through the specified `Gateway` resource. + + To deploy a WAF on multiple gateways, you must create a separate `EnvoyExtensionPolicy` resource for each `Gateway` resource. + Each `EnvoyExtensionPolicy` must reference the same `tigera-waf-backend` backend. + +1. To verify that the WAF is enabled for your gateway, you can simulate an SQL injection attack through your gateway and check that the WAF logs the request. + + :::note + The query string in this example has some SQL syntax embedded in the text. This is harmless and for demo purposes, but WAF will detect this pattern and create a WAF log for this HTTP request. By design, Calico Ingress Gateway WAF emits WAF logs rather than security events. + ::: + + 1. Get the service IP of your gateway: + + ```bash + export GATEWAY_HOST=$(kubectl get gateway/ -o jsonpath='{.status.addresses[0].value}') + ``` + 1. Simulate an SQL injection attack on that service IP: + + ```bash + curl --verbose --header "Host: www.example.com" http://$GATEWAY_HOST/?artist=0+div+1+union%23foo*%2F*bar%0D%0Aselect%23foo%0D%0A1%2C2%2Ccurrent_user + ``` + 1. In **Logs**, select the `tigera_secure_ee_waf*` index pattern and look for a WAF log that corresponds with this request. If blocking mode is enabled, the `curl` command also returns a `403 Forbidden` response. + +## Customizing your WAF configuration for an ingress gateway + +The default WAF configuration uses the OWASP Core Rule Set v4.7.0 in detection-only mode with early blocking enabled. +To customize a WAF configuration for an ingress gateway, you can include custom directives in an `EnvoyPatchPolicy` resource. + +For information on how to create custom directives, see [WAF customization options](web-application-firewall.mdx#customization-options). + +For example, to enable the OWASP Core Rule Set in blocking mode, you can apply an `EnvoyPatchPolicy` resource like this: + +```yaml +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyPatchPolicy +metadata: + name: tigera-waf-http-filter-metadata + namespace: +spec: + targetRef: + group: gateway.networking.k8s.io + kind: Gateway + name: + type: JSONPatch + jsonPatches: + - type: "type.googleapis.com/envoy.config.listener.v3.Listener" + name: //http + operation: + op: add + jsonPath: "default_filter_chain.filters[0].typed_config.http_filters[?match(@.name, '.*tigera-waf-envoy-extension-policy/.*')]" + path: typed_config/grpc_service/initial_metadata + value: + - key: directivesJson + value: "[\"SecRuleEngine On\"]" +``` + +You can add additional directives to the `directivesJson` array to further customize the WAF behavior. +Make sure that combine the directives as an inlined JSON list with proper escaping. +The array in the following snippet is an example of how to set the WAF to detection-only mode, remove a specific rule, and set a sampling percentage: + +```yaml + - key: directivesJson + value: "[\"SecRuleEngine DetectionOnly\", \"SecRuleRemoveById 920420\", \"SecAction \\\""id:900400,phase:1,pass,nolog,setvar:tx.sampling_percentage=50\\\"\]" +``` + +## Additional resources + +* [OWASP ModSecurity](https://modsecurity.org/) +* [OWASP Core Rule Set documentation](https://coreruleset.org/) \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/attach-waf-to-a-route.mdx b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/attach-waf-to-a-route.mdx new file mode 100644 index 0000000000..662c70870b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/attach-waf-to-a-route.mdx @@ -0,0 +1,108 @@ +--- +description: Attach a WAFPolicy to a Gateway or HTTPRoute to turn on WAF inspection for that target, layering namespace overrides on the cluster baseline. +--- + +# Attach WAF to a Gateway or route + +:::note + +This feature is tech preview. Tech preview features may be subject to significant changes before they become GA. + +::: + +Create a `WAFPolicy` in your namespace pointing at Gateway API targets — WAF inspection doesn't start automatically when the feature is enabled. It inherits the cluster baseline with namespace overrides layered on top. + +## Prerequisites + +- The cluster operator has created the `GlobalWAFPolicy` baseline. See [Set a cluster-wide baseline for Gateway WAF](./set-a-baseline.mdx). +- You have write access to `WAFPolicy` in your namespace. +- A `Gateway` or `HTTPRoute` in your namespace is already receiving traffic. + +## Choose a target + +`WAFPolicy.spec.targetRefs[]` accepts two kinds of targets: + +| `kind` in `targetRefs` | WAF covers | Use when | +| :--- | :--- | :--- | +| `Gateway` | Every `HTTPRoute` bound to that Gateway | You want uniform WAF across all routes on a Gateway | +| `HTTPRoute` | Just that one route | You need per-route tuning — different paranoia level, different plugins | + +Each `targetRefs` entry generates a separate `EnvoyExtensionPolicy` (max 16 entries). + +:::note + +`sectionName` scopes a `targetRefs` entry to a specific listener on a `Gateway` (or a named section), so WAF applies to just that listener rather than the whole Gateway. + +::: + +## Attach a WAFPolicy + +1. Create a `WAFPolicy` in your namespace pointing at the target. This example attaches to one `HTTPRoute`, overriding action and paranoia; the unset `coreRuleSet.state` still inherits from the baseline: + + ```yaml + apiVersion: applicationlayer.projectcalico.org/v3 + kind: WAFPolicy + metadata: + name: payments-api-waf + namespace: payments + spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: payments-api + action: Block # override: Block even if global baseline is Detect + coreRuleSet: + paranoiaLevel: 2 # stricter than the cluster default + # coreRuleSet.state is unset → inherited from GlobalWAFPolicy + ``` + + Apply it with `kubectl apply -f wafpolicy.yaml`. For inheritance details, see the [Overview](./overview.mdx#inheritance-namespaces-override-they-dont-restart). + +1. Verify the policy was accepted and programmed: + + ```bash + kubectl get wafpolicy payments-api-waf -n payments + ``` + + ``` + NAME ACCEPTED PROGRAMMED AGE + payments-api-waf True True 45s + ``` + + `.status.conditions` reports `Accepted`, `Programmed`, and validation findings. Confirm Envoy Gateway received the filter with `kubectl get envoyextensionpolicies -n payments` (one per `targetRefs` entry). + +1. Test that the WAF blocks an attack: + + ```bash + curl -si "https:///api?id=1+OR+1%3D1" + ``` + + ``` + HTTP/1.1 403 Forbidden + content-length: 11 + content-type: text/plain + + Access denied + ``` + + :::tip + + In `Detect` mode the request is allowed (200), but the WAF hit is still recorded in the `tigera_secure_ee_waf` log index (carrying `gateway_name`/`gateway_namespace`). On this release, gateway-WAF hits are not raised as Security Events — see [Observing WAF activity](./troubleshooting.mdx#observing-waf-activity). + + ::: + + :::caution + + `ACCEPTED=False` reason `ValidationFailed` means a `GlobalWAFValidationPolicy` is blocking your config — the status message names the policy and rule. Contact your cluster operator or see [Validate configuration with Rego](./validate-config.mdx). + + ::: + +## Conflict resolution + +Only one `WAFPolicy` may be active per target. If two in the same namespace target the same Gateway/HTTPRoute, $[prodname] follows [GEP-713](https://gateway-api.sigs.k8s.io/geps/gep-713/): oldest `creationTimestamp` wins; the newer is marked `Accepted=False` reason `Conflicted`, and only the winner's directives are merged. Fix by deleting the newer policy or repointing its `targetRefs`. + +## Next steps + +- [Write custom rules with plugins](./write-custom-rules.mdx) — add app-specific detections beyond the baseline. +- [Validate configuration with Rego](./validate-config.mdx) — check the merged config against compliance guardrails. +- [Troubleshooting](./troubleshooting.mdx) — diagnose a policy that won't accept or program. diff --git a/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/enable.mdx b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/enable.mdx new file mode 100644 index 0000000000..4d7fe96645 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/enable.mdx @@ -0,0 +1,76 @@ +--- +description: Turn Gateway WAF on or off by setting the operator GatewayAPI CR's extensions.waf.state field. +--- + +# Enable Gateway WAF + +:::note + +This feature is tech preview. Tech preview features may be subject to significant changes before they become GA. + +::: + +Gateway WAF is off by default. Turning it on is a one-line change to the operator's `GatewayAPI` CR — no new workload is deployed, so there's no extra resource footprint to plan for. + +## Prerequisites + +- Your cluster is connected to $[prodname]. +- Calico Ingress Gateway is already set up. See [About Calico Ingress Gateway](../../networking/ingress-gateway/about-calico-ingress-gateway.mdx). +- Cluster-admin rights — the `GatewayAPI` CR is a cluster-scoped operator resource. + +:::note + +Customer-installed (BYO) Envoy Gateway is not supported. Gateway WAF requires the Ingress Gateway that $[prodname] manages. + +::: + +## Enable Gateway WAF + +1. Set `spec.extensions.waf.state` to `Enabled` on the `GatewayAPI` resource named `tigera-secure`, which you created when you set up the ingress gateway. + The default is `Disabled`. + + ```bash + kubectl patch gatewayapi tigera-secure --type merge -p '{"spec":{"extensions":{"waf":{"state":"Enabled"}}}}' + ``` + + The operator installs the six WAF CRDs and the validating admission webhook, and wires the WAF reconcilers into the existing `calico-kube-controllers` pod in `calico-system`. No new workload is deployed. + +1. Confirm the six CRDs are installed: + + ```bash + kubectl get crd | grep applicationlayer.projectcalico.org + ``` + + ```text + globalwafplugins.applicationlayer.projectcalico.org + globalwafpolicies.applicationlayer.projectcalico.org + globalwafvalidationpolicies.applicationlayer.projectcalico.org + wafplugins.applicationlayer.projectcalico.org + wafpolicies.applicationlayer.projectcalico.org + wafvalidationpolicies.applicationlayer.projectcalico.org + ``` + + All six must be present before you create any WAF resources. + +1. Confirm the admission webhook is active: + + ```bash + kubectl get validatingwebhookconfigurations | grep waf + ``` + + You should see one entry. If it's absent, check the `calico-kube-controllers` pod logs in the `calico-system` namespace. + +## Disable Gateway WAF + +Set `spec.extensions.waf.state` back to `Disabled` on the same `GatewayAPI` resource to turn the feature off: + +```bash +kubectl patch gatewayapi tigera-secure --type merge -p '{"spec":{"extensions":{"waf":{"state":"Disabled"}}}}' +``` + +The operator removes the admission webhook and stops the WAF reconcilers; existing `WAFPolicy`, `GlobalWAFPolicy`, and other WAF resources stay on the cluster but are no longer reconciled or enforced until you re-enable the feature. + +## Next steps + +- [Tutorial](./tutorial.mdx) — deploy a test app and block your first attack. +- [Set a cluster-wide baseline for Gateway WAF](./set-a-baseline.mdx) — start your production rollout. diff --git a/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/multi-tenant-setup.mdx b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/multi-tenant-setup.mdx new file mode 100644 index 0000000000..66c3c9eaf8 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/multi-tenant-setup.mdx @@ -0,0 +1,89 @@ +--- +description: Run a full multi-tenant Gateway WAF setup — a cluster baseline, org-wide plugin, and validation policy that stops teams weakening protection while each team tunes independently. +--- + +# Multi-tenant setup + +:::note + +This feature is tech preview. Tech preview features may be subject to significant changes before they become GA. + +::: + +Multi-tenant setup ties the baseline, plugins, and validation guardrails together so a shared cluster stays safe by default while each team still controls its own Gateway. + +:::note[Scenario] + +A cluster shared by `team-a` and `payments`, with a cluster operator setting the baseline via a `GlobalWAFPolicy`, an org-wide `GlobalWAFPlugin`, and a `GlobalWAFValidationPolicy` that stops teams weakening protection. Each team attaches WAF to its own Gateway and tunes independently. + +::: + +## Prerequisites + +- Gateway WAF is enabled. See [Enable Gateway WAF](./enable.mdx). +- Cluster-admin rights for the cluster-operator steps; each team needs write access to WAF resources in its own namespace. + +## Set it up + +1. As cluster operator, create the baseline resources: the `default` `GlobalWAFPolicy`, an `org-security-baseline` `GlobalWAFPlugin`, and a `production-requirements` `GlobalWAFValidationPolicy` (`Enforce`, scoped to `env: production`). Follow the patterns in [Set a cluster-wide baseline](./set-a-baseline.mdx), [Write custom rules](./write-custom-rules.mdx), and [Validate configuration with Rego](./validate-config.mdx). + +1. Grant application operators read access to the `Global*` resources so they can see what they're inheriting: + + ```yaml + apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: waf-global-reader + rules: + - apiGroups: ["applicationlayer.projectcalico.org"] + resources: + - globalwafpolicies + - globalwafplugins + - globalwafvalidationpolicies + verbs: ["get", "list", "watch"] + --- + apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: waf-global-reader + namespace: team-a # repeat for payments + subjects: + - kind: Group + name: team-a-operators + apiGroup: rbac.authorization.k8s.io + roleRef: + kind: ClusterRole + name: waf-global-reader + apiGroup: rbac.authorization.k8s.io + ``` + + :::note + + This grants read on cluster-scoped resources via a namespace-scoped `RoleBinding` — application operators can see but not edit `Global*` resources. + + ::: + +1. Each team labels its namespace so the validation policy applies (`kubectl label namespace team-a env=production`, same for `payments`), then creates a `WAFPolicy` targeting its own Gateway. `team-a` raises paranoia to 2, inheriting `coreRuleSet.state: Enabled`; `payments` runs paranoia 3 plus a namespace `WAFPlugin` blocking GraphQL introspection, using an ID from its own [SecRule ID range](./reference.mdx#secrule-id-ranges). + +## What happens if a team tries to weaken the baseline + +If `payments` applies `action: Detect`, `production-requirements` (`Enforce`) fires its `must-use-block-mode` critical rule — the controller marks the policy `Programmed=False, Reason=NotAttempted` and surfaces the violation: + +```bash +kubectl get wafpolicy payments-waf -n payments \ + -o jsonpath='{.status.validation.failures}' | jq +``` + +No `EnvoyExtensionPolicy` is generated until corrected — restore `action: Block` or request an exception from the cluster operator. + +## Result: isolation in practice + +- **`team-a`** — Core Rule Set at paranoia 2, own custom rules, own ID range. +- **`payments`** — Core Rule Set at paranoia 3, GraphQL introspection block, a different ID range. +- Neither team can disable the org-wide rule from `GlobalWAFPlugin` or drop below Block mode. +- The cluster operator sees all violations centrally on `GlobalWAFValidationPolicy/production-requirements` status — one place to see which namespaces are non-compliant. + +## Next steps + +- [Reference](./reference.mdx) — CRDs, Rego validation, SecRule ID ranges, and status conditions. +- [Troubleshooting](./troubleshooting.mdx) — diagnose a policy blocked by a guardrail. diff --git a/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/overview.mdx b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/overview.mdx new file mode 100644 index 0000000000..2355c45ea0 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/overview.mdx @@ -0,0 +1,199 @@ +--- +description: Protect Calico Ingress Gateway traffic with a per-route and per-namespace web application firewall built on the OWASP Core Rule Set — how the resource families, merge order, and enforcement guardrails fit together. +--- + +# Gateway WAF + +:::note + +This feature is tech preview. Tech preview features may be subject to significant changes before they become GA. + +::: + +## Big picture + +Protect the traffic entering your cluster through Calico Ingress Gateway with a web application firewall (WAF) that you attach per Gateway or per route, and tune per namespace. + +## Value + +Edge WAFs traditionally apply one ruleset to everything behind the gateway. Gateway WAF takes a Kubernetes-native, multi-tenant approach: a cluster operator sets a security baseline once, and each application team layers its own rules and exceptions on top — scoped to their own Gateways and `HTTPRoute`s, within guardrails the cluster operator defines. + +Gateway WAF runs the industry-standard [OWASP ModSecurity Core Rule Set v4](https://coreruleset.org/) as a [Coraza](https://coraza.io/) WebAssembly filter inside Envoy Gateway — no sidecars, and no traffic ever leaves the proxy for inspection. + +## How it works + +Gateway WAF is configured entirely through Kubernetes resources. The controller merges every policy and plugin that applies to a target, generates an ordered list of ModSecurity directives, validates the result, and programs it into Envoy Gateway as an `EnvoyExtensionPolicy` that loads the Coraza WASM filter. + +Gateway WAF request flow. GlobalWAFPolicy/WAFPolicy and GlobalWAFPlugin/WAFPlugin are merged into an ordered list of SecLang directives. Rego validation then either passes the result to an EnvoyExtensionPolicy that loads the Coraza WASM filter on Envoy Gateway, or, on an Enforce-mode critical violation, programs no filter and reports the violation on status. The admission webhook validates submitted rules at apply time. + +The inputs to the merge are `GlobalWAFPolicy` + `WAFPolicy` and `GlobalWAFPlugin` + `WAFPlugin`. Two guardrails apply: the admission webhook (structural, at write time) and validation policies (compliance, after the merge). + +## Who does what + +Gateway WAF is configured through six custom resources in the `applicationlayer.projectcalico.org/v3` API group, in three families. Each family splits cleanly across two personas: a cluster-scoped resource owned by the **cluster operator (CO)**, and a namespace-scoped resource owned by the **application operator (AO)**. + +| Family | Cluster operator (CO) | Application operator (AO) | Answers the question | +| :--- | :--- | :--- | :--- | +| **Policy** | `GlobalWAFPolicy` — cluster-wide baseline (singleton, `name: default`) | `WAFPolicy` — attach WAF to a Gateway/`HTTPRoute` (`targetRefs`); per-namespace overrides | *What* to enforce — action, Core Rule Set state, paranoia, which plugins | +| **Plugin** | `GlobalWAFPlugin` — organization-wide custom rules | `WAFPlugin` — app-specific rules and false-positive exceptions | *Which custom rules* to add, in ModSecurity SecLang | +| **Validation** | `GlobalWAFValidationPolicy` — compliance guardrails (`Enforce`/`Audit`) | `WAFValidationPolicy` — advisory self-check, never blocks | *Whether a configuration is allowed* — compliance guardrails | + +Enabling the feature itself (the `GatewayAPI` gate) is a one-time cluster operator step — see [Enable the feature](./enable.mdx) — everything above is ongoing configuration once it's on. + +The cluster operator *sets* the posture; the application operator *tunes* it — can go stricter, but can be stopped from going weaker (see [Validation and enforcement](#validation-and-enforcement)). + +## Policies — what to enforce + +A **policy** declares the action mode (`Detect`/`Block`), whether the Core Rule Set is enabled, the paranoia level, and which plugins to load. + +- `GlobalWAFPolicy` — cluster-wide singleton (`name: default`), sets the baseline for every namespace. +- `WAFPolicy` — namespace-scoped, attaches WAF to a Gateway/`HTTPRoute` via `targetRefs`, can override the baseline action and paranoia. + +## Plugins — custom rules + +A **plugin** holds custom ModSecurity rules in four lifecycle slots — `config`, `before`, `rules`, `after` — modeled on the [Core Rule Set 4 plugin system](https://coreruleset.org/docs/4-about-plugins/4-1-plugins/). Referenced from a policy's `spec.plugins[]`; does nothing on its own. + +- `GlobalWAFPlugin` — organization-wide rules: security baselines, compliance rules. +- `WAFPlugin` — application-specific rules and false-positive exceptions. + +Each slot runs at a fixed point in the merge, so slot choice decides when your rules fire: + +| Slot | Runs | Use for | +| :--- | :--- | :--- | +| `config` | Before Core Rule Set setup | `tx.*` variable setup (anomaly thresholds, optional features, app flags). Rules on request variables belong in `before` instead — the request isn't read yet | +| `before` | After variable setup, before Core Rule Set loads | Early-exit allowlisting of trusted paths, request normalization, flags for later rules | +| `rules` | Alongside Core Rule Set, after it (Tigera extension) | App-specific detections that complement rather than pre-empt the Core Rule Set; can act on anomaly scores | +| `after` | Last | False-positive suppression, response inspection, logging. `GlobalWAFPlugin.after` runs *after* namespace `WAFPlugin.after`, so the cluster operator has final say | + +For the full merge sequence across every policy and plugin, see [merged ruleset load order](#the-merged-ruleset-load-order); to write plugin rules, see [Write custom rules](./write-custom-rules.mdx). + +## Validation policies — guardrails + +A **validation policy** carries [Rego](https://www.openpolicyagent.org/docs/latest/policy-language/) rules evaluated against the *merged, effective* configuration — answering "is this acceptable?", e.g. "Core Rule Set must stay enabled" or "production must run in Block mode." + +- `GlobalWAFValidationPolicy` — enforced by the cluster operator: `Enforce` mode blocks programming on a `critical` violation, `Audit` mode only warns. `namespaceSelector` scopes which namespaces it applies to. +- `WAFValidationPolicy` — advisory only, surfaces violations on status for the application operator to act on, never blocks. No enforce mode at namespace scope. + +(Validation policies aren't the admission webhook — see [Validation and enforcement](#validation-and-enforcement) for the distinction.) + +## The global policy is a singleton + +`GlobalWAFPolicy` is a cluster-wide singleton — must be named `default`. It sets the baseline (default action, Core Rule Set state, paranoia level) every namespace inherits unless it overrides. + +`GlobalWAFPlugin` and `GlobalWAFValidationPolicy` are **not** singletons — create as many as needed. + +## Inheritance: namespaces override, they don't restart + +A `WAFPolicy` doesn't have to restate the whole baseline — its `action` and `coreRuleSet` fields are optional; unset fields inherit from `GlobalWAFPolicy`. + +```yaml +apiVersion: applicationlayer.projectcalico.org/v3 +kind: WAFPolicy +metadata: + name: payments-waf + namespace: payments +spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: payments-api + action: Block # override the global default for this namespace + coreRuleSet: + paranoiaLevel: 2 # stricter than the baseline + # coreRuleSet.state is unset → inherited from GlobalWAFPolicy +``` + +| `WAFPolicy` field | When set | When unset | +| :--- | :--- | :--- | +| `action` | overrides the global `defaultAction` for this namespace | inherits `GlobalWAFPolicy.spec.defaultAction` | +| `coreRuleSet.state` | overrides the global state | inherits the global state | +| `coreRuleSet.paranoiaLevel` | overrides the global level | inherits the global level | + +## Plugins are additive, not overriding + +Plugins stack rather than override. The merger applies all referenced global plugins first (`GlobalWAFPolicy.spec.plugins[]` order), then all namespace plugins (`WAFPolicy.spec.plugins[]` order) — order is array position; there's no priority field. + +This lets baseline rules and application exceptions coexist in one ruleset — the interleaving, and why global `after` runs last, is covered below. + +## Merge order and enforcement + +When a `WAFPolicy` attaches to a target, the controller gathers every applicable policy and plugin, merges them into one ordered list of ModSecurity directives, validates the result, and programs it into Envoy Gateway as an `EnvoyExtensionPolicy`. Each `targetRefs[]` entry produces its own `EnvoyExtensionPolicy` — three routes means three filters. + +Three things govern the outcome: **merge order**, the **admission webhook**, and **validation policies**. + +### The merged ruleset load order + +Directive order isn't cosmetic: the Core Rule Set needs variables set before loading, and post-processing after. Fixed order: + +**Engine setup** +1. `SecRuleEngine On` / `DetectionOnly` — the effective action mode from the merged config. +2. Engine settings — audit logging and request-body access. + +**Plugin `config` (variable setup)** +3. Global plugin `config` blocks, in `GlobalWAFPolicy.spec.plugins[]` order. +4. Namespace plugin `config` blocks, in `WAFPolicy.spec.plugins[]` order. + +**Core Rule Set setup** +5. Core Rule Set setup — reads the `tx.*` variables set above. +6. Blocking paranoia level, from the merged paranoia level. +7. Detection paranoia level (always ≥ blocking level). + +**Pre-Core Rule Set rules (`before`)** +8. Global plugin `before` rules. +9. Namespace plugin `before` rules. + +**Core Rule Set** +10. The full OWASP Core Rule Set. + +**Custom rules (`rules`)** +11. Global plugin `rules`. +12. Namespace plugin `rules`. + +**Post-processing (`after`)** +13. Namespace plugin `after` rules. +14. Global plugin `after` rules — **runs last, giving the cluster operator the final say** on anomaly scoring and logging. + +Pattern: global-before-namespace in the early slots, namespace-before-global in `after`. Within a plugin, its own slots always run `config` → `before` → `rules` → `after`. + +:::note + +When two `WAFPolicy` resources target the same route, $[prodname] follows Gateway API [GEP-713](https://gateway-api.sigs.k8s.io/geps/gep-713/): oldest `creationTimestamp` wins (ties by `namespace/name`). The loser is marked `Accepted=False` reason `Conflicted`, directives not merged. + +::: + +### Structural safety: the admission webhook + +Before any of the above runs, a validating admission webhook checks the raw SecLang in a `WAFPlugin` or `WAFPolicy` at apply time, rejecting the resource if any directive fails one of three checks: + +| Check | Passes when | Rejected with | +| :--- | :--- | :--- | +| Directive is allowed | the directive type is on the allowlist (`SecRule`, `SecAction`, `SecRuleRemoveById`, and other safe directives) | `UnsafeDirective` | +| Rule ID is in range | a `SecRule`/`SecAction` ID falls inside the namespace's allocated ID range | `IDOutsideRange` | +| Removal is permitted | a `SecRuleRemoveById` targets an ID inside the namespace's own range, not a protected baseline rule | `ProtectedIDRemoval` | + +This stops one namespace colliding with another's rule IDs, or disabling the cluster's baseline Core Rule Set rules. Each namespace gets an automatic 10,000-ID range; see [SecRule ID ranges](./reference.mdx#secrule-id-ranges) for how it's derived and which IDs are protected. + +### Validation and enforcement + +The admission webhook checks *raw SecLang*; validation policies check the *merged result*, running after the merge because some problems only show up there. Two innocent plugins can combine to disable all SQL-injection rules — invisible to admission-time checks, but visible to a Rego rule against the merged directive list. + +Validation rules are [Rego](https://www.openpolicyagent.org/docs/latest/policy-language/) modules with severity `info`, `warning`, or `critical`. What happens on a violation depends on the policy: + +| Validation policy | Mode | `critical` violation | `warning` / `info` violation | +| :--- | :--- | :--- | :--- | +| `GlobalWAFValidationPolicy` | `Enforce` | **blocks** — no filter is programmed; status reports the rejection | warns on status; filter still programmed | +| `GlobalWAFValidationPolicy` | `Audit` (default) | warns on status; filter still programmed | warns on status; filter still programmed | +| `WAFValidationPolicy` | advisory only | surfaces on `WAFPolicy` status; never blocks | surfaces on status; never blocks | + +To write the rules, see [Rego validation](./reference.mdx#rego-validation). To read the outcome, see [Status conditions](./reference.mdx#status-conditions). + +## Next steps + +- [Enable the feature](./enable.mdx) — turn on Gateway WAF and verify the CRDs and webhook. +- [Tutorial](./tutorial.mdx) — deploy a test app and block your first attack. +- [Set a baseline](./set-a-baseline.mdx), [Attach WAF](./attach-waf-to-a-route.mdx), [Write custom rules](./write-custom-rules.mdx), [Validate config](./validate-config.mdx), [Multi-tenant setup](./multi-tenant-setup.mdx) — production tasks. +- [Reference](./reference.mdx) — CRDs, Rego validation, SecRule ID ranges, and status conditions. +- [Troubleshooting](./troubleshooting.mdx) — diagnose common failure modes. diff --git a/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/reference.mdx b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/reference.mdx new file mode 100644 index 0000000000..bda3529e2e --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/reference.mdx @@ -0,0 +1,376 @@ +--- +description: Field reference for the six Gateway WAF custom resources, the Rego validation input schema, SecRule ID range allocation, and status conditions. +--- + +# Reference + +:::note + +This feature is tech preview. Tech preview features may be subject to significant changes before they become GA. + +::: + +## CRDs + +Six custom resources in the `applicationlayer.projectcalico.org/v3` API group, in three families; each has a cluster-scoped and namespace-scoped resource sharing almost all fields. + +| Resource | Scope | Short name | Persona | Purpose | +| :--- | :--- | :--- | :--- | :--- | +| `GlobalWAFPolicy` | Cluster | `gwafp` | CO | Cluster-wide baseline (singleton, `name: default`) | +| `WAFPolicy` | Namespace | `wafp` | AO | Attach WAF to a target; per-namespace overrides | +| `GlobalWAFPlugin` | Cluster | `gwafplugin` | CO | Organization-wide custom rules | +| `WAFPlugin` | Namespace | `wafplugin` | AO | App-specific rules and exceptions | +| `GlobalWAFValidationPolicy` | Cluster | `gwafvp` | CO | Compliance guardrails (enforce or audit) | +| `WAFValidationPolicy` | Namespace | `wafvp` | AO | Advisory self-validation | + +### Policy family + +`GlobalWAFPolicy` sets the cluster baseline; `WAFPolicy` attaches WAF to a target and overrides the baseline for its namespace. + +```yaml +apiVersion: applicationlayer.projectcalico.org/v3 +kind: GlobalWAFPolicy +metadata: + name: default # singleton: name MUST be "default" +spec: + defaultAction: Block # Detect | Block + coreRuleSet: + state: Enabled # Enabled | Disabled + paranoiaLevel: 1 # 1 (balanced) to 4 (paranoid) + plugins: # order = array position + - kind: GlobalWAFPlugin + name: security-baseline +``` + +`WAFPolicy` adds `targetRefs` (`Gateway`/`HTTPRoute`); `action` and `coreRuleSet` are optional overrides of the global defaults: + +```yaml +apiVersion: applicationlayer.projectcalico.org/v3 +kind: WAFPolicy +metadata: + name: payments-waf + namespace: payments +spec: + targetRefs: # attach to a Gateway and/or HTTPRoute + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: payments-api + - group: gateway.networking.k8s.io + kind: Gateway + name: team-gateway + action: Block # optional; nil = inherit global defaultAction + coreRuleSet: + paranoiaLevel: 2 # optional; nil = inherit + plugins: + - kind: WAFPlugin + name: api-exceptions +``` + +**Field reference — Policy family** + +| Field | Type | Resource | Notes | +| :--- | :--- | :--- | :--- | +| `defaultAction` | `Detect` \| `Block` | GlobalWAFPolicy | Cluster default; defaults to `Detect` | +| `action` | `Detect` \| `Block` | WAFPolicy | Optional; **nil = inherit** the global default | +| `coreRuleSet.state` | `Enabled` \| `Disabled` | both | Defaults to `Enabled`; nil on WAFPolicy = inherit | +| `coreRuleSet.paranoiaLevel` | integer 1–4 | both | Defaults to `1`; nil on WAFPolicy = inherit | +| `plugins[]` | list of `{kind, name}` | both | `kind`: `WAFPlugin` \| `GlobalWAFPlugin`; order = array position; max 64 | +| `targetRefs[]` | Gateway API target refs | **WAFPolicy only** | `kind`: `Gateway` \| `HTTPRoute`; min 1, max 16. `sectionName` optionally scopes a ref to a specific listener/section | + +### Plugin family + +`GlobalWAFPlugin`/`WAFPlugin` share an identical spec: custom ModSecurity rules in four lifecycle slots — see [Write custom rules](./write-custom-rules.mdx). + +```yaml +apiVersion: applicationlayer.projectcalico.org/v3 +kind: GlobalWAFPlugin +metadata: + name: security-baseline +spec: + description: "Cluster security baseline" + config: | # 1. set tx.* variables + SecAction "id:1234001,phase:1,pass,nolog,setvar:tx.enforce_https=1" + before: | # 2. pre-CRS processing + SecRule REQUEST_URI "@beginsWith /webhooks/" \ + "id:1234010,phase:1,pass,nolog,ctl:ruleRemoveById=942200" + rules: | # 3. custom rules alongside CRS + SecRule REQUEST_HEADERS:User-Agent "@pmFromFile bad-bots.data" \ + "id:1234020,phase:1,deny,status:403,msg:'Blocked bot'" + after: | # 4. post-CRS processing + SecAction "id:1234030,phase:5,pass,nolog,log" +``` + +`WAFPlugin` uses the identical spec at namespace scope — only `metadata.namespace` and the persona differ. + +**Field reference — Plugin family** + +| Field | Type | Notes | +| :--- | :--- | :--- | +| `description` | string | Human-readable purpose; max 1024 chars | +| `config` | string (SecLang) | Slot 1 — `tx.*` variable setup, runs before Core Rule Set setup; max 65536 chars | +| `before` | string (SecLang) | Slot 2 — pre-Core Rule Set rules; max 65536 chars | +| `rules` | string (SecLang) | Slot 3 — custom rules alongside Core Rule Set (Tigera extension); max 65536 chars | +| `after` | string (SecLang) | Slot 4 — post-Core Rule Set rules; max 65536 chars | + +Rule IDs must fall in your namespace's allocated range — see [SecRule ID ranges](#secrule-id-ranges). + +### Validation family + +`GlobalWAFValidationPolicy` can enforce; `WAFValidationPolicy` is advisory only. Both carry Rego rules — see [Rego validation](#rego-validation). + +```yaml +apiVersion: applicationlayer.projectcalico.org/v3 +kind: GlobalWAFValidationPolicy +metadata: + name: security-requirements +spec: + enforcementMode: Enforce # Enforce | Audit (default Audit) + namespaceSelector: # optional; empty = all namespaces + matchLabels: + env: production + rules: + - name: crs-must-be-enabled + severity: critical + rego: | + package waf + violations[{"msg": "OWASP CRS must be enabled"}] { + input.config.crsState != "Enabled" + } +``` + +`WAFValidationPolicy` is namespace-scoped, advisory only — no `enforcementMode`, no `namespaceSelector`; every rule runs as if in `Audit` mode. + +**Field reference — Validation family** + +| Field | Type | Resource | Notes | +| :--- | :--- | :--- | :--- | +| `enforcementMode` | `Audit` \| `Enforce` | **GlobalWAFValidationPolicy only** | Defaults to `Audit`. `Enforce` blocks on a `critical` violation | +| `namespaceSelector` | label selector | **GlobalWAFValidationPolicy only** | Empty = all namespaces | +| `rules[]` | list of `ValidationRule` | both | At least one required | +| `rules[].name` | string | both | 1–253 chars | +| `rules[].rego` | string (Rego module) | both | Must declare `package waf` and a `violations` rule; max 16384 chars | +| `rules[].severity` | `info` \| `warning` \| `critical` | both | Defaults to `warning` | + +--- + +## Rego validation + +A validation policy carries [Rego](https://www.openpolicyagent.org/docs/latest/policy-language/) rules evaluated after the configuration is merged — so a rule can reason about the final action, paranoia level, and merged directive list, regardless of which policy or plugin contributed them. + +### The input document + +Every rule evaluates against the same `input` document — the merged configuration and its provenance: + +| Path | Type | Meaning | +| :--- | :--- | :--- | +| `input.config.action` | string | Effective action after merge — `Detect` or `Block` | +| `input.config.crsState` | string | Core Rule Set state — `Enabled` or `Disabled` | +| `input.config.paranoiaLevel` | integer | Effective paranoia level, 1–4 | +| `input.directives` | array of string | The complete, ordered list of merged ModSecurity directives | +| `input.source.globalPolicy` | string | Name of the global policy applied (always `default`) | +| `input.source.namespacePolicy` | string | Name of the `WAFPolicy`, if any | +| `input.source.globalPlugins` | array of string | Global plugin names, in load order | +| `input.source.namespacePlugins` | array of string | Namespace plugin names, in load order | + +### The violations contract + +A rule's `rego` field must declare `package waf` and a `violations` rule producing objects with at least a `msg` field. No violations means compliant; each object is a finding, surfaced on status, with any extra fields passed through. + +```rego +package waf # required — the engine evaluates this package + +# Each block that adds to `violations` is one check. +violations[{"msg": msg}] { # a violation is an object with a msg field + input.config.crsState != "Enabled" # the condition that is NOT allowed + msg := "OWASP CRS must be enabled" +} + +violations[{"msg": msg}] { + input.config.paranoiaLevel < 2 # body lines are ANDed; all must hold + msg := sprintf("paranoia level %d is below the minimum of 2", [input.config.paranoiaLevel]) +} +``` + +Or inspect the merged directive list directly — useful for catching a plugin that removes a baseline rule: + +```rego +package waf + +violations[{"msg": msg}] { + d := input.directives[_] # iterate the merged directives + startswith(d, "SecRuleRemoveById") + re_match(`94[0-9]{4}`, d) # a CRS SQL-injection rule (942xxx) + msg := sprintf("removes a baseline SQLi rule: %s", [d]) +} +``` + +### Worked examples + +How different merged configurations evaluate against the rules above: + +| Effective configuration | `violations` | +| :--- | :--- | +| `crsState: Enabled`, `paranoiaLevel: 2` | empty — compliant | +| `crsState: Disabled` | `"OWASP CRS must be enabled"` | +| `paranoiaLevel: 1` | `"paranoia level 1 is below the minimum of 2"` | +| a plugin with `SecRuleRemoveById 942100` | `"removes a baseline SQLi rule: ..."` | + +### Severity + +Each rule has a `severity` — `info`, `warning`, `critical` (default `warning`) — deciding the consequence, not whether it fires: + +- In `Enforce` mode, a `critical` violation blocks the configuration from being programmed. +- In `Audit` mode — and in any `WAFValidationPolicy` — violations are reported on status but never block. + +See the enforcement table in [How it works](./overview.mdx#validation-and-enforcement) for the full matrix. + +--- + +## SecRule ID ranges + +Every custom rule in a `WAFPlugin` carries a numeric ModSecurity ID. Gateway WAF allocates each namespace a fixed ID range and blocks any namespace from disabling baseline rules — both enforced by the admission webhook. + +### How a namespace's range is derived + +Each namespace is allocated a contiguous block of 10,000 IDs, derived deterministically from the namespace name: + +```text +prefix = first four hex digits of sha256(namespace name), modulo 10000 +range = [ prefix * 10000 , prefix * 10000 + 9999 ] +``` + +The range is a pure function of the namespace name — it never changes and needs no cluster state. It isn't a globally unique reservation: two namespaces can map to the same block, harmlessly, since each runs its own WAF engine and neither can touch the protected ranges below. + +### Finding your range + +The first time a namespace references a plugin, the controller stamps the range onto it as an annotation: + +```bash +kubectl get namespace payments \ + -o jsonpath='{.metadata.annotations.applicationlayer\.projectcalico\.org/waf-id-range}' +``` + +Pick the IDs for your `WAFPlugin` rules from inside this range. + +### Protected ranges + +Some IDs are reserved and rejected for every namespace: + +- **`900000`–`999999`** — the OWASP Core Rule Set baseline. +- **Tigera-shipped baseline rules** — the rules that implement the managed security baseline. + +A namespace can neither define rules with these IDs nor remove them with `SecRuleRemoveById` — stopping a tenant from quietly disabling cluster protection. + +### What the admission webhook rejects + +Applying a `WAFPlugin`/`WAFPolicy` triggers the webhook to parse every directive; failing any check below rejects the resource, with the reason on its `Accepted` condition. + +| Check | Rejected with | Meaning | +| :--- | :--- | :--- | +| Directive not on the allowlist | `UnsafeDirective` | The directive type isn't permitted (for example, one that reads files or fetches remote rules) | +| Rule ID outside your range | `IDOutsideRange` | A `SecRule`/`SecAction` ID falls outside the namespace's allocated range | +| Removing a protected rule | `ProtectedIDRemoval` | A `SecRuleRemoveById` targets an ID outside the namespace's range (such as a Core Rule Set baseline rule) | + +The controller re-runs these checks at reconcile as a backstop, marking `Accepted=False` with the matching reason instead of programming. + +### Allowed directives + +Allowed: `SecRule`, `SecAction`, `SecDefaultAction`, `SecRuleRemoveById`, `SecRuleRemoveByTag`, `SecRuleUpdateActionById`, `SecRuleUpdateTargetById`, `SecMarker`, and request/response body and argument limits. Denied: directives that read the filesystem, write audit logs to arbitrary paths, or fetch remote rules. + +### Example + +A namespace whose range is `420000`–`429999` writes rules using IDs from that block: + +```yaml +apiVersion: applicationlayer.projectcalico.org/v3 +kind: WAFPlugin +metadata: + name: api-exceptions + namespace: payments +spec: + rules: | + SecRule REQUEST_BODY "@rx __schema" \ + "id:420020,phase:2,deny,status:403,msg:'Introspection blocked'" +``` + +--- + +## Status conditions + +Gateway WAF reports policy state through standard Kubernetes status conditions: + +```bash +kubectl describe wafpolicy payments-waf -n payments +# or +kubectl get wafpolicy payments-waf -n payments -o jsonpath='{.status.conditions}' | jq +``` + +A healthy policy has `Licensed=True`, `Accepted=True`, `Validated=True`, `Programmed=True`, `Ready=True`, evaluated roughly in that order — an early failure explains why later conditions are absent or `False`. + +### Licensed + +This condition reports whether the cluster is entitled to Gateway WAF. + +| Reason | Meaning | +| :--- | :--- | +| `LicenseValid` | The cluster is entitled to this feature | +| `LicenseRequired` | No valid entitlement; the policy is not programmed | + +### Accepted + +Whether the resource is structurally valid and admitted. Several reasons come straight from the [admission webhook](#what-the-admission-webhook-rejects). + +| Reason | Meaning | +| :--- | :--- | +| `Accepted` | The resource is valid and admitted | +| `Invalid` | The spec or a directive is invalid (the message says which) | +| `ValidationFailed` | A `GlobalWAFValidationPolicy` in `Enforce` mode reported a `critical` violation | +| `Conflicted` | Another policy already targets this route; the older one wins (the message names it) | +| `PluginNotFound` | A referenced `WAFPlugin`/`GlobalWAFPlugin` does not exist | +| `UnsafeDirective` | A directive is not on the allowlist | +| `IDOutsideRange` | A rule ID is outside the namespace's allocated range | +| `ProtectedIDRemoval` | A `SecRuleRemoveById` targets a protected baseline rule | + +### Validated + +The result of evaluating [validation policies](#rego-validation) against the merged configuration. + +| Reason | Meaning | +| :--- | :--- | +| `Compliant` | No violations | +| `ViolationsFound` | One or more rules reported a violation (see `status.validation.failures`) | +| `NotEvaluated` | No validation policy applies to this resource | + +### Programmed + +Whether the Coraza WASM filter was programmed into Envoy Gateway. + +| Reason | Meaning | +| :--- | :--- | +| `ConfigurationApplied` | The `EnvoyExtensionPolicy` was generated and the filter is active | +| `TargetNotFound` | No valid `targetRefs`, or a referenced `Gateway`/`HTTPRoute` does not exist | +| `RenderTooLarge` | The merged ruleset exceeds the render size limit | +| `WASMUnavailable` | The Coraza WASM image could not be pulled | +| `NotAttempted` | A `critical` violation under `Enforce` blocked programming | + +### Ready + +A roll-up of the conditions above. + +| Reason | Meaning | +| :--- | :--- | +| `Ready` | The policy is active on the data plane | +| `NotReady` | One of the conditions above is not satisfied | + +### Validation summary + +`WAFPolicy.status.validation` summarizes validation without reading individual conditions: + +| Field | Meaning | +| :--- | :--- | +| `status` | `Valid`, `Audited`, or `Rejected` | +| `securityPosture` | `Compliant`, `Warning`, `Degraded`, or `Critical` | +| `failures[]` | Per-rule failures — policy name, rule, severity, and message | +| `lastEvaluated` | When validation last ran | + +A `GlobalWAFValidationPolicy`'s own status aggregates violating namespaces cluster-wide — one place to see who's out of compliance. diff --git a/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/set-a-baseline.mdx b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/set-a-baseline.mdx new file mode 100644 index 0000000000..986410a075 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/set-a-baseline.mdx @@ -0,0 +1,112 @@ +--- +description: Set the cluster-wide WAF baseline every namespace inherits — action mode, Core Rule Set state, and paranoia level. +--- + +# Set a cluster-wide baseline for Gateway WAF + +:::note + +This feature is tech preview. Tech preview features may be subject to significant changes before they become GA. + +::: + +The baseline is the cluster-wide default every namespace inherits — action mode, Core Rule Set, and paranoia level. Set it once so every Gateway starts protected and teams tune from a known-safe starting point. + +## Prerequisites + +- Gateway WAF is enabled. See [Enable Gateway WAF](./enable.mdx). +- Cluster-admin rights — `GlobalWAFPolicy` is cluster-scoped. + +## Set the baseline + +1. Create the singleton `GlobalWAFPolicy`. It must be named `default` — other names are rejected by the admission webhook. Start in `Detect` mode: it logs matches without blocking, so you can measure false positives before switching to `Block`: + + ```yaml + apiVersion: applicationlayer.projectcalico.org/v3 + kind: GlobalWAFPolicy + metadata: + name: default + spec: + defaultAction: Detect # start here; promote to Block after validating false-positive rate + coreRuleSet: + state: Enabled + paranoiaLevel: 1 + ``` + + Apply it with `kubectl apply -f globalwafpolicy.yaml`. + +1. Pick a paranoia level. Higher catches more attacks at the cost of more false positives: + + | Level | Posture | Typical use | + | :--- | :--- | :--- | + | 1 | Balanced — low false-positive rate | General-purpose default; a safe starting point | + | 2 | Moderate — catches common evasion patterns | APIs that handle sensitive data | + | 3 | Strict — flags most suspicious patterns | High-value or compliance-regulated endpoints | + | 4 | Paranoid — near-zero tolerance | Extremely high-security targets; expect false positives | + + :::tip + + Start at level 1 cluster-wide; let application operators raise it per namespace, and enforce a minimum later via a validation policy. + + ::: + +1. Once you've validated the false-positive rate, set `defaultAction: Block`. $[prodname] then returns HTTP 403 for matching requests. + +## Add organization-wide rules with a GlobalWAFPlugin (optional) + +A `GlobalWAFPlugin` holds custom ModSecurity rules applied cluster-wide via the global policy's `plugins[]` list — for a security baseline every namespace inherits. + +1. Create the `GlobalWAFPlugin`. Rules use the four lifecycle slots described in [Write custom rules](./write-custom-rules.mdx): + + ```yaml + apiVersion: applicationlayer.projectcalico.org/v3 + kind: GlobalWAFPlugin + metadata: + name: org-baseline + spec: + description: "Organization-wide security baseline rules" + config: | + SecAction "id:800001,phase:1,pass,nolog,setvar:tx.enforce_https=1" + before: | + SecRule REQUEST_HEADERS:X-Forwarded-Proto "!@streq https" \ + "id:800002,phase:1,deny,status:403,msg:'HTTPS required',chain" + SecRule TX:ENFORCE_HTTPS "@eq 1" "" + rules: | + SecRule REQUEST_HEADERS:X-Request-ID "@rx ^$" \ + "id:800003,phase:1,deny,status:400,msg:'Missing X-Request-ID header'" + after: | + # Cluster operator's after slot runs last in the merge order. + SecRule TX:BLOCKING_ANOMALY_SCORE "@gt 0" \ + "id:800099,phase:5,pass,log,msg:'Request blocked by WAF'" + ``` + +1. Reference it from `GlobalWAFPolicy.spec.plugins[]` — for example `plugins: [{kind: GlobalWAFPlugin, name: org-baseline}]`. `plugins[]` is ordered by array position, up to 64 entries; `WAFPlugin` entries can be added here too. + +## Verify the baseline + +1. Confirm the policy is accepted: + + ```bash + kubectl get globalwafpolicy default + ``` + + ``` + NAME ACCEPTED AGE + default True 30s + ``` + +Detail lives in `.status.conditions` (`kubectl get globalwafpolicy default -o yaml`): `Accepted=True` reason `Accepted` = programmed; `Accepted=False` reason `ValidationFailed` = a validation policy blocked it (the message describes the violation). See [Status conditions](./reference.mdx#status-conditions). + +Every namespace with a `WAFPolicy` inherits `defaultAction`, `coreRuleSet.state`, and `coreRuleSet.paranoiaLevel` from `GlobalWAFPolicy` unless it overrides them: + +``` +GlobalWAFPolicy (baseline) + └─ WAFPolicy in namespace payments → overrides action: Block, paranoiaLevel: 2 + └─ WAFPolicy in namespace team-a → inherits everything from GlobalWAFPolicy +``` + +## Next steps + +- [Attach WAF to a Gateway or route](./attach-waf-to-a-route.mdx) — turn on inspection for a target. +- [Write custom rules with plugins](./write-custom-rules.mdx) — add app-specific detections and exceptions. +- [Validate configuration with Rego](./validate-config.mdx) — enforce compliance guardrails on the merged config. diff --git a/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/troubleshooting.mdx b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/troubleshooting.mdx new file mode 100644 index 0000000000..0e0f9b983c --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/troubleshooting.mdx @@ -0,0 +1,200 @@ +--- +description: Diagnose and fix common Gateway WAF problems by reading policy status conditions and working through symptom-to-fix guidance for each failure mode. +--- + +# Troubleshooting + +:::note + +This feature is tech preview. Tech preview features may be subject to significant changes before they become GA. + +::: + +Most problems surface on the policy's status conditions: + +```bash +kubectl describe wafpolicy -n +``` + +A healthy policy has `Licensed=True`, `Accepted=True`, `Validated=True`, `Programmed=True`, `Ready=True` — evaluated roughly in order, so an early failure explains later absences. See [Status conditions](./reference.mdx#status-conditions) for reason meanings. + +## Observing WAF activity + +WAF hits (both `Block` and `Detect`/`would_block`) are recorded in the `tigera_secure_ee_waf` log index, carrying `gateway_name`/`gateway_namespace` for the originating Gateway. On this release they are **not** raised as Security Events: the intrusion-detection controller excludes WAF logs that carry a gateway name, so gateway-WAF activity is observed through the WAF log index rather than the Security Events view. Re-surfacing gateway WAF as Security Events (or on a dedicated dashboard) is planned for a later release. + +Raw Envoy proxy logs are a last-resort diagnostic when a hit is missing from the index. + +--- + +## `Accepted=False, Reason=Conflicted` + +**Cause.** Two `WAFPolicy` resources in the same namespace target the same Gateway/HTTPRoute. Per [GEP-713](https://gateway-api.sigs.k8s.io/geps/gep-713/), oldest `creationTimestamp` wins; the newer is marked `Conflicted`, directives never merged. + +**Fix.** Identify the competing policies: + +```bash +kubectl get wafpolicy -n -o wide +``` + +Delete one, or repoint its `targetRefs` — see [How it works](./overview.mdx#merge-order-and-enforcement) for why only one policy owns a target. + +--- + +## `Accepted=False, Reason=UnsafeDirective | IDOutsideRange | ProtectedIDRemoval` + +**Cause.** The admission webhook rejected the SecLang in a `WAFPlugin` or `WAFPolicy` at apply time, before the resource reached the controller. + +| Reason | Meaning | +| :--- | :--- | +| `UnsafeDirective` | The directive type is not on the allowlist (for example, a directive that reads from the filesystem) | +| `IDOutsideRange` | A `SecRule` or `SecAction` ID falls outside the namespace's allocated 10,000-ID range | +| `ProtectedIDRemoval` | A `SecRuleRemoveById` targets a protected ID — a Core Rule Set baseline rule or a Tigera-managed rule | + +**Fix.** Read the rejection message on `status.conditions`: + +```bash +kubectl get wafpolicy -n \ + -o jsonpath='{.status.conditions[?(@.type=="Accepted")].message}' +``` + +For `IDOutsideRange`/`ProtectedIDRemoval`: use only IDs from your namespace's range: + +```bash +kubectl get namespace \ + -o jsonpath='{.metadata.annotations.applicationlayer\.projectcalico\.org/waf-id-range}' +``` + +For `UnsafeDirective`: remove or replace the directive. See [SecRule ID ranges](./reference.mdx#secrule-id-ranges) for the allowlist. + +--- + +## `Programmed=False, Reason=WASMUnavailable` + +**Cause.** The Envoy proxy couldn't pull the Coraza WASM image — usually a registry issue: an unreachable reference, or a missing pull secret. + +**Diagnostic.** Check the `EnvoyExtensionPolicy` status the controller generated: + +```bash +kubectl get envoyextensionpolicy \ + -n \ + -l applicationlayer.projectcalico.org/waf-policy= \ + -o jsonpath='{.items[*].status}' | jq +``` + +Check Envoy proxy logs for `could not fetch Wasm OCI image`: + +```bash +kubectl logs -n \ + -l gateway.envoyproxy.io/owning-gateway-namespace= \ + -c envoy | grep -i wasm +``` + +**Fix.** The operator manages the image reference. In a private registry, wire in the pull secret via operator configuration, and confirm the node can reach the registry with the secret attached to the Envoy proxy pods. + +--- + +## `Programmed=False, Reason=NotAttempted` + +**Cause.** A `GlobalWAFValidationPolicy` in `Enforce` mode found a `critical` violation, so the controller never programmed the Coraza filter. + +**Diagnostic.** Read the validation failures: + +```bash +kubectl get wafpolicy -n \ + -o jsonpath='{.status.validation}' | jq +``` + +`failures[]` names the failed rule, with severity and message; `status.validation.status` is `Rejected`. + +**Fix.** Satisfy the failing rule — e.g. restore `action: Block` if the policy requires it — or ask the cluster operator for an exception. See [Validate configuration with Rego](./validate-config.mdx) and [Status conditions](./reference.mdx#status-conditions). + +--- + +## `Accepted=False, Reason=TargetNotFound` + +**Cause.** `targetRefs` references a `Gateway`/`HTTPRoute` that doesn't exist in the namespace. + +**Fix.** Verify the target exists: + +```bash +kubectl get gateway,httproute -n +``` + +Check `name`, `kind`, `group` in `targetRefs` match the resource, then create it and re-apply, or fix the reference — reconciliation is automatic once it exists. + +--- + +## Attacks not blocked + +**Cause.** One or more of: + +- `action` is `Detect` (explicit or inherited from `GlobalWAFPolicy`) — inspected, not blocked. +- `coreRuleSet.state` is `Disabled` — Core Rule Set isn't loaded. +- `coreRuleSet.paranoiaLevel` is below the triggering rule's paranoia floor — the rule doesn't evaluate. + +**Diagnostic.** Check the policy's conditions and spec: + +```bash +kubectl describe wafpolicy -n +``` + +Confirm action is `Block` and Core Rule Set `Enabled`. If paranoia looks right but the attack still passes, check whether that rule's floor is higher than configured — some only activate at paranoia 2 or 3. + +:::tip + +`kubectl waf` (coming in a future release) will render the full merged directive list so you can inspect the effective `SecRuleEngine` line and paranoia settings directly. + +::: + +Also check the `GlobalWAFPolicy` default action, inherited when a namespace `action` is unset: + +```bash +kubectl get globalwafpolicy default -o jsonpath='{.spec.defaultAction}' +``` + +See [How it works](./overview.mdx#inheritance-namespaces-override-they-dont-restart) for field inheritance rules. + +--- + +## `403` on every request / false positives + +**Cause.** The paranoia level is too high for the traffic pattern, or a specific Core Rule Set rule is matching legitimate requests (a false positive). + +**Diagnostic.** The firing rule ID and message are in the `tigera_secure_ee_waf` log index for the hit. As a last resort, the Envoy proxy access logs in the Gateway pod also carry them: + +```bash +kubectl logs -n \ + -l gateway.envoyproxy.io/owning-gateway-namespace= \ + -c envoy | grep -i "waf\|403" | tail -30 +``` + +**Fix.** Two options: + +1. **Lower the paranoia level**, if false positives come from a broad class of higher-paranoia rules and your threat model allows it. +2. **Add a plugin exception** — suppress via `SecRuleRemoveById`, or skip a known-safe target via `SecRuleUpdateTargetById`. Preferred for surgical tuning without weakening the overall posture. + +For how to write exception rules within your namespace's ID range, see [Write custom rules with plugins](./write-custom-rules.mdx). + +:::caution + +`SecRuleRemoveById` can only target rule IDs within your namespace's allocated range. You cannot remove Core Rule Set baseline rules (IDs `900000`–`999999`) or Tigera-managed rules — the admission webhook will reject the plugin with `ProtectedIDRemoval`. + +::: + +--- + +## General diagnostic sequence + +If no specific symptom matches above, walk the condition chain in order: + +```bash +kubectl get wafpolicy -n \ + -o jsonpath='{range .status.conditions[*]}{.type}={.status} ({.reason}): {.message}{"\n"}{end}' +``` + +1. `Licensed=False` → the cluster isn't entitled to Gateway WAF. + Check that your $[prodname] subscription tier includes Gateway WAF, or [contact Support](../../get-help/support.mdx). +2. `Accepted=False` → spec rejected. Read the `Reason`/`message` — see the sections above. +3. `Validated=False` → a validation rule fired. Read `status.validation.failures`. +4. `Programmed=False` → filter not loaded into Envoy. Read the `Reason` — see `WASMUnavailable`/`NotAttempted` above. +5. All conditions `True` but not blocking → give Envoy a few seconds to load the WASM module after `EnvoyExtensionPolicy` creation; if it still doesn't block, the effective action is likely `Detect` — see [Attacks not blocked](#attacks-not-blocked) above. diff --git a/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/tutorial.mdx b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/tutorial.mdx new file mode 100644 index 0000000000..8311bba4b6 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/tutorial.mdx @@ -0,0 +1,236 @@ +--- +description: Deploy a test app behind Calico Ingress Gateway, set a WAF baseline, attach it to the Gateway, and confirm a SQL injection and XSS attack are both blocked. +--- + +# Tutorial + +:::note + +This feature is tech preview. Tech preview features may be subject to significant changes before they become GA. + +::: + +This tutorial takes you from a running Ingress Gateway to a blocked attack in about five minutes. + +## Prerequisites + +- Gateway WAF is enabled. See [Enable Gateway WAF](./enable.mdx). +- Calico Ingress Gateway is set up. See [About Calico Ingress Gateway](../../networking/ingress-gateway/about-calico-ingress-gateway.mdx). +- A namespace where you can deploy a test app — this tutorial creates one called `httpbin`. + +## Block an attack in five minutes + +1. Deploy a test application. Create an `httpbin` namespace with a deployment and service to act as the backend: + + ```yaml + apiVersion: v1 + kind: Namespace + metadata: + name: httpbin + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: httpbin + namespace: httpbin + spec: + replicas: 1 + selector: + matchLabels: + app: httpbin + template: + metadata: + labels: + app: httpbin + spec: + containers: + - name: httpbin + image: kong/httpbin:0.1.0 + ports: + - containerPort: 80 + --- + apiVersion: v1 + kind: Service + metadata: + name: httpbin + namespace: httpbin + spec: + selector: + app: httpbin + ports: + - protocol: TCP + port: 80 + targetPort: 80 + ``` + + Apply it, then wait for the pod to be ready: + + ```bash + kubectl wait --for=condition=Ready pod -l app=httpbin -n httpbin --timeout=60s + ``` + +1. Create a Gateway and HTTPRoute: + + ```yaml + apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + name: httpbin-gateway + namespace: httpbin + spec: + gatewayClassName: tigera-gateway-class + listeners: + - name: http + protocol: HTTP + port: 9080 + --- + apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + name: httpbin-route + namespace: httpbin + spec: + parentRefs: + - name: httpbin-gateway + rules: + - matches: + - path: + type: PathPrefix + value: / + backendRefs: + - name: httpbin + port: 80 + ``` + + :::note + + Use port 9080, not 8080. Port 8080 is reserved by the Calico L7 log collector inside the Envoy proxy pod. + + ::: + + Apply it, then wait for the Gateway to be accepted: + + ```bash + kubectl wait --for=condition=Accepted gateway/httpbin-gateway -n httpbin --timeout=60s + ``` + + At this point `Accepted` only means Gateway passed validation. Let's now wait for the Envoy proxy + Deployment that Ingress Gateway creates: + + ```bash + kubectl wait --for=condition=Available deployment \ + -l gateway.envoyproxy.io/owning-gateway-name=httpbin-gateway \ + -n httpbin --timeout=180s + ``` + +1. Confirm traffic flows. Find the Envoy service the Ingress Gateway created for this Gateway (the service lands in the Gateway's own namespace, `httpbin`): + + ```bash + export GATEWAY_SVC=$(kubectl get svc -n httpbin \ + -o jsonpath='{.items[?(@.metadata.labels.gateway\.envoyproxy\.io/owning-gateway-name=="httpbin-gateway")].metadata.name}') + ``` + + Send a test request from inside the cluster: + + ```bash + kubectl run curl-test -n httpbin --rm -it --restart=Never --image=curlimages/curl -- \ + curl -s -o /dev/null -w "%{http_code}" \ + http://$GATEWAY_SVC.httpbin.svc.cluster.local:9080/get + ``` + + ```text + 200 + ``` + + Traffic is flowing through the Gateway with no WAF protection yet. + +1. Apply a `GlobalWAFPolicy` to set the cluster-wide baseline. Create the singleton `default` policy with the OWASP Core Rule Set enabled in `Block` mode: + + :::tip + + This tutorial uses `Block` mode immediately so you can see blocking in action. Most production rollouts start in `Detect` mode to baseline traffic before switching. See [Merge order and enforcement](./overview.mdx#merge-order-and-enforcement) for the tradeoffs. + + ::: + + ```yaml + apiVersion: applicationlayer.projectcalico.org/v3 + kind: GlobalWAFPolicy + metadata: + name: default + spec: + defaultAction: Block + coreRuleSet: + state: Enabled + paranoiaLevel: 1 + ``` + + This defines the global posture. It does not attach WAF to any traffic yet — that requires a namespace-scoped `WAFPolicy` with `targetRefs`. + +1. Attach WAF to the Gateway. Create a `WAFPolicy` in the `httpbin` namespace targeting the Gateway: + + ```yaml + apiVersion: applicationlayer.projectcalico.org/v3 + kind: WAFPolicy + metadata: + name: httpbin-waf + namespace: httpbin + spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: Gateway + name: httpbin-gateway + ``` + + The reconciler merges the `GlobalWAFPolicy` baseline with this `WAFPolicy`, generates an ordered ModSecurity ruleset, and programs it into Envoy Gateway as an `EnvoyExtensionPolicy`. Check that the policy was accepted and programmed: + + ```bash + kubectl get wafpolicy httpbin-waf -n httpbin \ + -o jsonpath='{range .status.conditions[*]}{.type}={.status} ({.reason}){"\n"}{end}' + ``` + + ```text + Licensed=True (LicenseValid) + Accepted=True (Accepted) + Validated=Unknown (NotEvaluated) + Programmed=True (ConfigurationApplied) + Ready=True (Ready) + ``` + + All four executable conditions (`Licensed`, `Accepted`, `Programmed`, `Ready`) must be `True` before WAF is active. `Validated=Unknown (NotEvaluated)` is expected when no `WAFValidationPolicy` is in scope. If `Programmed=False`, check the condition message — a common cause is the WASM image not being reachable (see [Troubleshooting](./troubleshooting.mdx)). Allow a few seconds for Envoy to load the WASM filter before running the tests below. + +1. Test WAF blocking. A normal request still passes, but a SQL injection probe and an XSS probe are both blocked: + + ```bash + # Normal request — expect 200 + kubectl run curl-ok -n httpbin --rm -it --restart=Never --image=curlimages/curl -- \ + curl -s -o /dev/null -w "%{http_code}" \ + http://$GATEWAY_SVC.httpbin.svc.cluster.local:9080/get + + # SQL injection probe — expect 403 + kubectl run curl-sqli -n httpbin --rm -it --restart=Never --image=curlimages/curl -- \ + curl -s -o /dev/null -w "%{http_code}" \ + "http://$GATEWAY_SVC.httpbin.svc.cluster.local:9080/get?id=1%27%20OR%20%271%27%3D%271" + + # XSS probe — expect 403 + kubectl run curl-xss -n httpbin --rm -it --restart=Never --image=curlimages/curl -- \ + curl -s -o /dev/null -w "%{http_code}" \ + "http://$GATEWAY_SVC.httpbin.svc.cluster.local:9080/get?q=%3Cscript%3Ealert(1)%3C%2Fscript%3E" + ``` + + The Coraza WASM filter, running inside Envoy, evaluated each request against the OWASP Core Rule Set. Core Rule Set rule group 942xxx detects SQL injection; group 941xxx detects cross-site scripting. Both attacks are blocked before the request reaches the application. + +## Clean up + +```bash +kubectl delete namespace httpbin +kubectl delete globalwafpolicies default +``` + +## Troubleshoot + +If a `WAFPolicy` you attach to a route later shows `Programmed=False` with reason `WASMUnavailable`, the Coraza WASM image could not be pulled. Private registries require a pull secret to be configured on the `GatewayAPI` CR. See [Troubleshooting](./troubleshooting.mdx) for diagnosis steps and the pull-secret configuration. + +## Next steps + +- [Set a cluster-wide baseline for Gateway WAF](./set-a-baseline.mdx) — start production configuration in `Detect` mode. +- [Reference](./reference.mdx) — CRDs, Rego validation, SecRule ID ranges, and status conditions. diff --git a/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/validate-config.mdx b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/validate-config.mdx new file mode 100644 index 0000000000..cfe79d08bd --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/validate-config.mdx @@ -0,0 +1,127 @@ +--- +description: Write Rego validation rules with GlobalWAFValidationPolicy and WAFValidationPolicy to enforce or audit compliance guardrails on the merged WAF configuration. +--- + +# Validate configuration with Rego + +:::note + +This feature is tech preview. Tech preview features may be subject to significant changes before they become GA. + +::: + +Gateway WAF has two guardrail layers: the admission webhook checks raw SecLang at apply time (structural safety); validation policies check the **merged result** at reconcile time, catching problems only visible once everything's merged. See [Validation and enforcement](./overview.mdx#validation-and-enforcement) for the matrix and [Rego validation](./reference.mdx#rego-validation) for the input schema. + +## Prerequisites + +- Gateway WAF is enabled. See [Enable Gateway WAF](./enable.mdx). +- Cluster-admin rights to create a `GlobalWAFValidationPolicy` (it's cluster-scoped). A `WAFValidationPolicy` only needs namespace write access. +- Rego rules evaluate against the merged config — see [Rego validation](./reference.mdx#rego-validation) for the input schema and worked examples. + +## Enforce guardrails cluster-wide with GlobalWAFValidationPolicy + +`GlobalWAFValidationPolicy` is cluster-scoped, applying to every namespace or a label-selected subset via `namespaceSelector`. It carries Rego rules and an `enforcementMode`: `Audit` (default) records violations while the filter stays programmed; `Enforce` blocks programming on a `critical` violation. `warning`/`info` violations are always recorded. + +1. Write the policy with Rego rules and an `enforcementMode`. This one requires the Core Rule Set enabled and `Block` mode for namespaces labeled `env: production`: + + ```yaml + apiVersion: applicationlayer.projectcalico.org/v3 + kind: GlobalWAFValidationPolicy + metadata: + name: production-requirements + spec: + enforcementMode: Enforce + namespaceSelector: + matchLabels: + env: production + rules: + + # critical: CRS must be enabled + - name: crs-must-be-enabled + severity: critical + rego: | + package waf + violations[{"msg": "OWASP CRS must be enabled"}] { + input.config.crsState != "Enabled" + } + + # critical: production namespaces must use Block mode + - name: block-mode-required + severity: critical + rego: | + package waf + violations[{"msg": "WAF must be in Block mode for production"}] { + input.config.action != "Block" + } + ``` + + For more rule patterns — a minimum paranoia level, or inspecting the merged directive list to catch a plugin removing a baseline rule — see [Rego validation](./reference.mdx#rego-validation). + +1. Start in `Audit` mode, review violations on existing `WAFPolicy` status, then switch to `Enforce` — so you don't block teams whose config predates the policy. + + Leave `namespaceSelector` empty for cluster-wide, or scope with `matchLabels`. Multiple policies can coexist — every matching rule is evaluated, and one `critical` violation (in `Enforce`) blocks programming. + +## Self-check your config with WAFValidationPolicy + +`WAFValidationPolicy` is namespace-scoped and advisory only — violations appear on status but never block programming. Use it as a self-check. + +1. Write a `WAFValidationPolicy` in your namespace: + + ```yaml + apiVersion: applicationlayer.projectcalico.org/v3 + kind: WAFValidationPolicy + metadata: + name: team-self-check + namespace: platform + spec: + rules: + + - name: graphql-plugin-active + severity: warning + rego: | + package waf + violations[{"msg": "graphql-waf plugin is not loaded — introspection unguarded"}] { + not "graphql-waf" in input.source.namespacePlugins + } + ``` + + Being advisory, it's safe to hold yourself to a stricter standard than the baseline — e.g. a higher minimum paranoia level. Violations surface on `WAFPolicy` status without interrupting traffic. + +## Read violation results + +Violations from matching policies appear in the affected `WAFPolicy`'s status conditions. + +1. Check the policy status: + + ```bash + kubectl get wafpolicy -n -o yaml + ``` + + A blocked policy shows the failing rule: + + ```yaml + status: + conditions: + - type: Ready + status: "False" + reason: ValidationFailed + message: > + GlobalWAFValidationPolicy/production-requirements rule "block-mode-required": + WAF must be in Block mode for production + - type: Accepted + status: "True" + ``` + +An audit violation instead leaves `Ready=True` and adds a `ValidationWarning=True` condition, reason `AuditViolation`, with a message like `rce-rules-intact" [warning]: RCE rule removal detected in merged config: SecRuleRemoveById 932100` (the filter stays programmed). See [Status conditions](./reference.mdx#status-conditions) for the full reference. + +:::tip + +Write Rego against the schema in [Rego validation](./reference.mdx#rego-validation); test locally with `opa eval` first. A syntax error invalidates the whole policy — status reports which rule failed to parse. + +::: + +## Next steps + +- [Multi-tenant setup](./multi-tenant-setup.mdx) — baseline, plugins, and guardrails working together on a shared cluster. +- [Reference](./reference.mdx) — the Rego input schema, worked examples, and status conditions. +- [Troubleshooting](./troubleshooting.mdx) — diagnose a policy blocked by validation. diff --git a/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/write-custom-rules.mdx b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/write-custom-rules.mdx new file mode 100644 index 0000000000..65a86b7a57 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/gateway-waf/write-custom-rules.mdx @@ -0,0 +1,139 @@ +--- +description: Write custom ModSecurity rules in a WAFPlugin or GlobalWAFPlugin, using the four lifecycle slots and your namespace's SecRule ID range. +--- + +# Write custom rules with plugins + +:::note + +This feature is tech preview. Tech preview features may be subject to significant changes before they become GA. + +::: + +A `WAFPlugin` (namespace) or `GlobalWAFPlugin` (cluster) holds custom [ModSecurity SecLang](https://github.com/owasp-modsecurity/ModSecurity/wiki/Reference-Manual-(v3.x)) directives in four lifecycle slots — `config`, `before`, `rules`, `after` — for app-specific detections and false-positive exceptions the Core Rule Set doesn't cover on its own. Cluster operators wanting the same rules org-wide use `GlobalWAFPlugin` instead — same fields, different scope. For what each slot does and when it runs, see [Plugins — custom rules](./overview.mdx#plugins--custom-rules). + +:::caution + +`ctl:ruleEngine=Off` inside a chained rule or per-request disables the engine for that transaction only. A top-level `SecRuleEngine Off` is rejected by the admission webhook. + +::: + +:::note + +`SecRuleRemoveById`/`ctl:ruleRemoveById` in `after` apply for the rest of phase processing — use to suppress your own false positives. Protected Core Rule Set baseline rules can't be removed from any slot; the webhook rejects the attempt. + +::: + +## Prerequisites + +- Gateway WAF is enabled. See [Enable Gateway WAF](./enable.mdx). +- You have write access to `WAFPlugin` in your namespace (or `GlobalWAFPlugin` at cluster scope). +- A `WAFPolicy` or `GlobalWAFPolicy` exists to reference the plugin from — a plugin does nothing until a policy loads it. See [Attach WAF to a Gateway or route](./attach-waf-to-a-route.mdx). + +## Write a plugin + +1. Find your namespace's SecRule ID range. Every `SecRule`/`SecAction` needs an ID, and each namespace gets a 10,000-ID block enforced at admission: + + ```bash + kubectl get namespace \ + -o jsonpath='{.metadata.annotations.applicationlayer\.projectcalico\.org/waf-id-range}' + # example output: 420000-429999 + ``` + + The range is populated the first time a plugin is referenced in your namespace. See [SecRule ID ranges](./reference.mdx#secrule-id-ranges) for protected ranges and the allowlist. + +1. Write the `WAFPlugin`, placing each rule in the slot that matches when it should fire. This example uses all four slots to protect a GraphQL API. The `4200xx` IDs below assume the `420000-429999` example range from the previous step — swap in IDs from your own namespace's range, or the admission webhook rejects the plugin with `IDOutsideRange`: + + ```yaml + apiVersion: applicationlayer.projectcalico.org/v3 + kind: WAFPlugin + metadata: + name: graphql-waf + namespace: platform + spec: + description: "GraphQL introspection guard and false-positive exceptions" + + config: | + # Flag the application type so later rules can branch on it. + SecAction \ + "id:420001,phase:1,pass,nolog,\ + setvar:tx.is_graphql=1" + + before: | + # Allowlist the schema endpoint used by the internal schema registry — + # skip WAF analysis entirely for that specific path. + SecRule REQUEST_URI "@streq /graphql/schema-registry" \ + "id:420010,phase:1,pass,nolog,\ + ctl:ruleEngine=Off,\ + tag:allowlist" + + rules: | + # Block introspection queries (production guard). + SecRule TX:IS_GRAPHQL "@eq 1" \ + "id:420020,phase:2,deny,status:403,\ + msg:'GraphQL introspection blocked',\ + logdata:'body_snippet=%{MATCHED_VAR}',\ + chain" + SecRule REQUEST_BODY "@rx __schema|__type" "" + + # Block suspicious user-agents known to scan GraphQL APIs. + SecRule REQUEST_HEADERS:User-Agent "@pmf graphql-scanners.dat" \ + "id:420021,phase:1,deny,status:403,\ + msg:'Known GraphQL scanner blocked',\ + logdata:'ua=%{MATCHED_VAR}'" + + after: | + # Remove a false positive: CRS rule 941130 fires on a legitimate + # Content-Type value used by our GraphQL client library. + SecRuleRemoveById 941130 + ``` + + :::tip + + `@pmf` loads a data file, which must be bundled into the Coraza WASM filter image — ask your cluster operator, or use `@pm` (inline phrase match) for a short string list. + + ::: + +1. Reference the plugin from a `WAFPolicy`, in `spec.plugins[]` (load order = array position): + + ```yaml + apiVersion: applicationlayer.projectcalico.org/v3 + kind: WAFPolicy + metadata: + name: platform-api-waf + namespace: platform + spec: + targetRefs: + - group: gateway.networking.k8s.io + kind: HTTPRoute + name: graphql-api + action: Block + plugins: + - kind: WAFPlugin + name: graphql-waf # ← your plugin above + ``` + + If the policy also references a `GlobalWAFPlugin`, its slots load before the namespace plugin's slots at each position. + + :::note + + Not all SecLang directives are permitted — directives that read the filesystem, fetch remote rules, or write to arbitrary audit log paths are rejected. See [SecRule ID ranges](./reference.mdx#secrule-id-ranges) for the allowlist. + + ::: + +1. Confirm acceptance and reconciliation: + + ```bash + kubectl get wafplugin graphql-waf -n platform -o yaml | grep -A10 "conditions:" + kubectl get wafpolicy platform-api-waf -n platform -o yaml | grep -A20 "conditions:" + ``` + + `Ready=True` on the `WAFPolicy` confirms the merged ruleset passed validation and the `EnvoyExtensionPolicy` was programmed. + +Gateway WAF runs the [OWASP Core Rule Set v4](https://coreruleset.org/docs/) via [Coraza](https://coraza.io/); the [ModSecurity reference manual](https://github.com/owasp-modsecurity/ModSecurity/wiki/Reference-Manual-(v3.x)) covers the full SecLang directive set. + +## Next steps + +- [Validate configuration with Rego](./validate-config.mdx) — enforce guardrails on the merged ruleset. +- [Reference](./reference.mdx) — SecRule ID ranges, allowed directives, and status conditions. +- [Troubleshooting](./troubleshooting.mdx) — diagnose a plugin that won't accept or program. diff --git a/calico-cloud_versioned_docs/version-24-1/threat/index.mdx b/calico-cloud_versioned_docs/version-24-1/threat/index.mdx new file mode 100644 index 0000000000..00d5122fcb --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/index.mdx @@ -0,0 +1,23 @@ +--- +description: Detect and respond to threats in Calico Cloud connected clusters with threat intelligence feeds, deep packet inspection, and WAF. +hide_table_of_contents: true +--- + +import { DocCardLink, DocCardLinkLayout } from '/src/___new___/components'; + + +# Threat defense + +Use real-time monitoring to detect and block threats to your cluster. + + + + + + + + + + + + diff --git a/calico-cloud_versioned_docs/version-24-1/threat/security-event-management.mdx b/calico-cloud_versioned_docs/version-24-1/threat/security-event-management.mdx new file mode 100644 index 0000000000..d9372972c4 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/security-event-management.mdx @@ -0,0 +1,122 @@ +--- +description: Triage and manage security events from Calico Cloud connected clusters in the Security Events Dashboard, with filtering, exceptions, and recommended remediation. +--- + +# Security event management + +Manage security events from your cluster in a single place. + +## Value + +Security events indicate that a threat actor may be present in your Kubernetes cluster. For example, a DNS request to a malicious hostname, a triggered WAF rule, or the opening of a sensitive file. $[prodname] provides security engineers and incident response teams with a single dashboard to manage threat alerts. Benefits include: + +- A filtered list of critical events with recommended remediation +- Identify impacts on applications +- Understand the scope and frequency of the issue +- Manage alert noise by dismissing events (show/hide) +- Manage alert noise by creating exceptions + +## Before you begin + +**Required** + +- [WAF is enabled](./web-application-firewall.mdx) + +**Limitations** + +- Only WAF basic security events. Over time, the dashboard will contain a full range of $[prodname] security events. +- You cannot control which users can view or edit the page using fine-grained role-based access controls + +## Security Events Dashboard + +The **Security Events Dashboard** page gives you a high-level view of recent security events. +You can use this visual reference to get an overall sense of your cluster's health. +If you find anything that merits further investigation, you can click on an event for more details. + +* In the web console, go to **Threat defense > Security Events Dashboard**. + +![Security Events Dashboard](/img/calico-enterprise/security-events-dashboard.png) + +## Security Events + +The **Security Events** page lists all the security events that have been detected for your cluster. +You can view and filter your security events to focus on + +* In the web console, go to **Threat Defense > Security Events**. + +### Dismiss a security event + +You can clear your security events list by dismissing events that you've finished reviewing. +When you dismiss an event, that event is no longer visible in the list. + +1. In the web console, go to **Threat Defense > Security Events**. +1. Find a security event in the list, and then click **Action > Dismiss Security Event**. + +### Create a security event exception + +You can prevent certain kinds of security events from appearing in the list by creating a security event exception. +This is helpful if you want to reduce alert noise for workloads that you know are safe. +When you create an exception, all matching security events are removed from the security events list. +Future matches will not appear in the list. + +1. In the web console, go to **Threat Defense > Security Events**. +1. Find a security event in the list, and then click **Action > Add exception**. +1. On the **Create an Exception** dialog, select a scope for the exception and click **Create Exception**. + +You can manage your exceptions by clicking **Threat Defense > Security Events > Exceptions**. +You can browse, edit, and delete exceptions on the list. + +### UI help + +**Event details page** + +Provides actions to remediate the detection and stop the attack from progressing. For example: + +![waf-security](/img/calico-enterprise/waf-security-events-latest.png) + +**Severity** + +$[prodname] calculates severity (Critical, High, Medium, Low) using a combination of NIST CVSS 3.0 and MITRE IDs. + +**MITRE IDs** + +Multiple MITRE IDs may be associated with a security event. + +**Attack Vector** +- Network +- Process +- File + +**MITRE Tactic** (based on the [MITRE tactics](https://attack.mitre.org/tactics/enterprise/)) includes a specific path, method, or scenario that can compromise cluster security. Valid entries: + +| Tactic | Target | Attack techniques | +| ------------------------------------------------------------ | ------------------------------- | ------------------------------------------------------------ | +| [Initial access](https://attack.mitre.org/tactics/TA0001/) | Network | Gain an initial foothold within a network using various entry vectors. | +| [Execution](https://attack.mitre.org/tactics/TA0002/) | Code in local or remove systems | Control code running on local or remote systems using malicious code. | +| [Impact](https://attack.mitre.org/tactics/TA0040/) | Systems and data | Disrupt availability or compromise integrity by manipulating business and operational processes. | +| [Persistence](https://attack.mitre.org/tactics/TA0003/) | Maintain footholds | Maintain access to systems across restarts, credential changes, and other interruptions. | +| [Privilege Escalation](https://attack.mitre.org/tactics/TA0004/) | Access permissions | Access higher-level permissions on a system or network. | +| [Defense Evasion](https://attack.mitre.org/tactics/TA0005/) | Avoid detection | Masquerade and hide malware to avoid detection to compromise software, data, scripts, and processes. | +| [Discovery](https://attack.mitre.org/tactics/TA0007/) | Determine your environment | Gain knowledge about your system and internal network. | + +### Frequently asked questions + +**How is the recommended remediation determined?** + +The Tigera Security Research team maps MITRE IDs to events and provides the recommended remediation. + +**Will I see all $[prodname] alerts in this dashboard?** + +No. $[prodname] security events do not encompass all types of alerts nor all security alert types; they only contain alerts for threats. Alerts for vulnerabilities detected in a container image, or misconfigurations in your Kubernetes cluster, are displayed in their respective dashboards. However, when vulnerabilities or misconfigurations are exploited by an attacker, those indicators of an attack are considered security events. + +**What does dismissing a security event do?** + +Dismissing a security event hides it from view. + +**Why are some fields in columns blank?** + +Security events generated from older managed clusters will not have values for the new fields (for example, MITRE IDs). You can dismiss these events. + +**Where can I view security event logs?** + +Go to **Logs** and select the `tigera_secure_ee_events` index. diff --git a/calico-cloud_versioned_docs/version-24-1/threat/suspicious-domains.mdx b/calico-cloud_versioned_docs/version-24-1/threat/suspicious-domains.mdx new file mode 100644 index 0000000000..983821f340 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/suspicious-domains.mdx @@ -0,0 +1,92 @@ +--- +description: Add threat intelligence feeds to Calico Cloud to detect DNS queries to suspicious domains from connected clusters and surface impacted pods in the anomaly dashboard. +--- + +# Trace and alert on suspicious domains + +## Big picture + +Add threat intelligence feeds to $[prodname] to trace DNS queries involving suspicious domains. + +## Value + +$[prodname] integrates with threat intelligence feeds so you can detect when endpoints in your Kubernetes clusters query DNS for suspicious domains, or receive answers with suspicious domains. When events are detected, an anomaly detection dashboard in the UI shows the full context, including which pod(s) were involved so you can analyze and remediate. + +## Concepts + +$[prodname] supports pull methods for updating threat feeds. Use this method for fully automated threat feed updates without user intervention. + +### Domain name threat feeds + +A best practice is to develop an allow-list of "known-good" domains that particular applications or services must access, and then [enforce this allow-list with network policy](../network-policy/domain-based-policy.mdx). + +In addition to allow-lists, you can use threat feeds to monitor your cluster for DNS queries to known malicious or suspicious domain names. $[prodname] monitors DNS queries and generates alerts for any that are listed in your threat feed. + +Threat feeds for domain names associated with malicious **egress** activity (e.g. command and control (C2) servers or data exfiltration), provide the most security value. Threat feeds that associate domain names with malicious **ingress** activity (e.g. port scans or IP sweeps) are less useful since these activities do not cause endpoints in your cluster to query DNS. It is better to consider [IP-based threat feeds](suspicious-ips.mdx) for ingress activity. + +## Before you begin... + +### Required + +Privileges to manage GlobalThreatFeed. + +### Recommended + +We recommend that you turn down the aggregation of DNS logs sent to Elasticsearch for configuring threat feeds. If you do not adjust DNS log aggregation settings, $[prodname] aggregates DNS queries from workloads in the same replica set. This means if a suspicious DNS query is detected, you will only know which replica set made the query and not which specific pod. Go to: [FelixConfiguration](../reference/resources/felixconfig.mdx) and set the field, **dnsLogsFileAggregationKind** to **0** to log individual pods separately. + +## How to + +This section describes how to pull threat feeds to $[prodname]. + +### Pull threat feed updates + +To add threat feeds to $[prodname] for automatic updates (default is once a day), the threat feed(s) must be available using HTTP(S), and return a newline-separated list of domain names. + +#### Using the web console + +1. From the web console, select **Threat Feeds** --> **Add Feed**. +2. Add your threat feed on the Add a New Threat Feed window. For example: + - **Feed Name**: feodo-tracker + - **Description**: This is my threat feed based on domains. +{/* vale Vale.Avoid = NO */} + - **URL**: https://my.threatfeed.com/deny-list +{/* vale Vale.Avoid = YES */} + - **Content type**: DomainNameSet + - **Labels**: Choose a label from the list. +3. Click **Save Changes**. + + From the **Action** menu, you can view or edit the details that you entered and can download the manifest file. + +> Go to the Security Events page to view events that are generated when an endpoint in the cluster queries a name on the list. For more information, see [Manage alerts](../observability/alerts.mdx). + +#### Using CLIs + +1. Create the GlobalThreatFeed YAML and save it to file. + The simplest example of this looks like the following. Replace the **name** and the **URL** with your feed. + + ```yaml + apiVersion: projectcalico.org/v3 + kind: GlobalThreatFeed + metadata: + name: my-threat-feed + spec: + content: DomainNameSet + mode: Enabled + description: 'This is my threat feed' + feedType: Custom + pull: + http: + url: https://my.threatfeed.com/deny-list + ``` + +2. Add the global threat feed to the cluster. + + ```shell + kubectl apply -f + ``` + +> Go to the Security Events page to view events that are generated when an endpoint in the cluster queries a name on the list. For more information, see [Manage alerts](../observability/alerts.mdx). + +## Additional resources + +See [GlobalThreatFeed](../reference/resources/globalthreatfeed.mdx) resource definition for all configuration options. diff --git a/calico-cloud_versioned_docs/version-24-1/threat/suspicious-ips.mdx b/calico-cloud_versioned_docs/version-24-1/threat/suspicious-ips.mdx new file mode 100644 index 0000000000..0314267b42 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/suspicious-ips.mdx @@ -0,0 +1,308 @@ +--- +description: Add threat intelligence feeds to Calico Cloud to alert on flows to suspicious IPs in connected clusters and optionally block them with a dynamic deny-list policy. +--- + +# Trace and block suspicious IPs + +## Big picture + +Add threat intelligence feeds to $[prodname] to trace network flows of suspicious IP addresses, and optionally block traffic to suspicious IPs. + +## Value + +$[prodname] integrates with threat intelligence feeds so you can detect when your Kubernetes clusters communicate with suspicious IPs. When communications are detected, an anomaly detection dashboard in the UI shows the full context, including which pod(s) were involved so you can analyze and remediate. You can also use a threat intelligence feed to power a dynamic deny-list, either to or from a specific group of sensitive pods, or your entire cluster. + +## Concepts + +$[prodname] supports pull methods for updating threat feeds. Use this method for fully automated threat feed updates without user intervention. + +### Suspicious IPs: test before you block + +There are many different types of threat intelligence feeds (community-curated, company-paid, and internally-developed) that you can choose to monitor in $[prodname]. We recommend that you assess the threat feed contents for false positives before blocking based on the feed. If you decide to block, test a subset of your workloads before rolling to production to ensure legitimate application traffic is not blocked. + +## Before you begin... + +### Required + +Privileges to manage GlobalThreatFeed and GlobalNetworkPolicy. + +### Recommended + +We recommend that you turn down the aggregation of flow logs sent to Elasticsearch for configuring threat feeds. If you do not adjust flow logs, $[prodname] aggregates over the external IPs for allowed traffic, and threat feed searches will not provide useful results (unless the traffic is denied by policy). Go to: [FelixConfiguration](../reference/resources/felixconfig.mdx) and set the field, **flowLogsFileAggregationKindForAllowed** to **1**. + +You can adjust the flow logs by running the following command: + +```bash +kubectl patch felixconfiguration default --type='merge' -p '{"spec":{"flowLogsFileAggregationKindForAllowed":1}}' +``` + +## How to + +This section describes how to pull threat feeds to $[prodname], and block traffic to a cluster for a suspicious IP. + +- [Pull threat feed updates](#pull-threat-feed-updates) +- [Block traffic to a cluster](#block-traffic-to-a-cluster) + +### Pull threat feed updates + +To add threat feeds to $[prodname] for automatic updates (default is once a day), the threat feed(s) must be available using HTTP(S), and return a newline-separated list of IP addresses or prefixes in CIDR notation. + +#### Using the web console + +1. From the web console, select **Threat Feeds** --> **Add Feed**. +2. Add your threat feed on the Add a New Threat Feed window. For example: + - **Feed Name**: feodo-tracker + - **Description**: This is the feodo-tracker threat feed. + - **URL**: [https://feodotracker.abuse.ch/downloads/ipblocklist.txt](https://feodotracker.abuse.ch/downloads/ipblocklist.txt) + - **Content type**: IPSet + - **Labels**: Choose a label from the list. +3. Click **Save Changes**. + + From the **Action** menu, you can view or edit the details that you entered and can download the manifest file. + +> Go to the Security Events page to view events that are generated when an IP is displayed on the threat feed list. For more information, see [Manage alerts](../observability/alerts.mdx). When you create a global threat feed in the web console, network traffic is not automatically blocked. If you find suspicious IPs on the Security Events page, you need to create a network policy to block the traffic. For help with policy, see [Block traffic to a cluster](#block-traffic-to-a-cluster). + +#### Using CLIs + +1. Create the GlobalThreatFeed YAML and save it to file. + The simplest example of this looks like the following. Replace the **name** and the **URL** with your feed. + + ```yaml + apiVersion: projectcalico.org/v3 + kind: GlobalThreatFeed + metadata: + name: my-threat-feed + spec: + content: IPSet + mode: Enabled + description: 'This is my threat feed' + feedType: Custom + pull: + http: + url: https://my.threatfeed.com/deny-list + ``` + +2. Add the global threat feed to the cluster. + + ```bash + kubectl apply -f + ``` + +> Go to the Security Events page to view events that are generated when an IP is displayed on the threat feed list. For more information, see [Manage alerts](../observability/alerts.mdx). + +### Block traffic to a cluster + +Create a new/edit existing threat feed to include the globalNetworkSet stanza, setting the labels you want to use to represent the deny-listed IPs. This stanza instructs $[prodname] to search for flows to and from the listed IP addresses, and maintain a GlobalNetworkSet containing the IP addresses. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalThreatFeed +metadata: + name: sample-global-threat-feed +spec: + content: IPSet + mode: Enabled + description: 'This is the sample global threat feed' + feedType: Custom + pull: + http: + url: https://an.example.threat.feed/deny-list + globalNetworkSet: + labels: + security-action: block +``` + +1. Add the global threat feed to the cluster. + + ```bash + kubectl apply -f + ``` + +2. Create a GlobalNetworkPolicy that blocks traffic based on the threat feed, by selecting sources or destinations using the labels you assigned in step 1. For example, the following GlobalNetworkPolicy blocks all traffic coming into the cluster if it came from any of the suspicious IPs. + + ```yaml + apiVersion: projectcalico.org/v3 + kind: GlobalNetworkPolicy + metadata: + name: default.blockthreats + spec: + tier: default + selector: all() + types: + - Ingress + ingress: + - action: Deny + source: + selector: security-action == 'block' + ``` + +3. Add the global network policy to the cluster. + +```bash + kubectl apply -f +``` + +## Tutorial + +In this tutorial, we’ll walk through setting up a threat feed to search for connections to suspicious IPs. Then, we’ll use the same threat feed to block traffic to those IPs. + +We will use the free [FEODO botnet tracker](https://feodotracker.abuse.ch/) from abuse.ch that lists IP addresses associated with command and control servers. But the steps are the same for your commercial or internal threat feeds. + +If you haven’t already adjusted your [aggregation flows](#before-you-begin), we recommend it before you start. + +### Configure the threat feed + +1. Create a file called feodo-tracker.yaml with the following contents: + + ```yaml + apiVersion: projectcalico.org/v3 + kind: GlobalThreatFeed + metadata: + name: feodo-tracker + spec: + content: IPSet + mode: Enabled + description: 'This is the feodo-tracker threat feed' + feedType: Custom + pull: + http: + url: https://feodotracker.abuse.ch/downloads/ipblocklist.txt + ``` + + This pulls updates using the default period of once per day. See the [Global Resource Threat Feed API](../reference/resources/globalthreatfeed.mdx) for all configuration options. + +2. Add the feed to your cluster. + + ```bash + kubectl apply -f feodo-tracker.yaml + ``` + +### Check search results + +Open the $[prodname] web console, and navigate to the “Security Events” page. If any of your pods have been communicating with the IP addresses in the FEODO tracker feed, you will see the results listed on this page. It is normal to not see any events listed on this page. + +### Block pods from contacting IPs + +If you have high confidence in the IP addresses listed as malicious in a threat feed, you can take stronger action than just searching for connections after the fact. For example, the FEODO tracker lists IP addresses used by command and control servers for botnets. We can configure $[prodname] to block all egress traffic to addresses on this list. + +It is strongly recommended that you assess the contents of a threat feed for false positives before using it as a deny-list, and that you apply it to a test subset of your workloads before rolling out application or cluster-wide. Failure to do so could cause legitimate application traffic to be blocked and could lead to an outage in your application. + +In this demo, we will apply the policy only to a test workload (so we do not impact other traffic). + +1. Create a file called **tf-ubuntu.yaml** with the following contents: + + ```yaml + apiVersion: v1 + kind: Pod + metadata: + labels: + docs.tigera.io-tutorial: threat-feed + name: tf-ubuntu + spec: + nodeSelector: + kubernetes.io/os: linux + containers: + - command: + - sleep + - '3600' + image: ubuntu + name: test + ``` + +2. Apply the pod configuration. + + ```bash + kubectl apply -f tf-ubuntu.yaml + ``` + +3. Edit the feodo-tracker.yaml to include a globalNetworkSet stanza: + + ```yaml + apiVersion: projectcalico.org/v3 + kind: GlobalThreatFeed + metadata: + name: feodo-tracker + spec: + content: IPSet + mode: Enabled + description: 'This is the feodo-tracker threat feed' + feedType: Custom + pull: + http: + url: https://feodotracker.abuse.ch/downloads/ipblocklist.txt + globalNetworkSet: + labels: + docs.tigera.io-threat-feed: feodo + ``` + +4. Reapply the new YAML. + + ```bash + kubectl apply -f feodo-tracker.yaml + ``` + +5. Verify that the GlobalNetworkSet is created. + + ```bash + kubectl get globalnetworksets threatfeed.feodo-tracker -o yaml + ``` + +### Apply global network policy + +We will now apply a GlobalNetworkPolicy that blocks the test workload from connecting to any IPs in the threat feed. + +1. Create a file called block-feodo.yaml with the following contents: + + ```yaml + apiVersion: projectcalico.org/v3 + kind: GlobalNetworkPolicy + metadata: + name: default.block-feodo + spec: + tier: default + selector: docs.tigera.io-tutorial == 'threat-feed' + types: + - Egress + egress: + - action: Deny + destination: + selector: docs.tigera.io-threat-feed == 'feodo' + - action: Allow + ``` + +2. Apply this policy to the cluster + + ```bash + kubectl apply -f block-feodo.yaml + ``` + +### Verify policy on test workload + +We will verify the policy from the test workload that we created earlier. + +1. Get a shell in the pod by running the following command: + + ```bash + kubectl exec -it tf-ubuntu -- bash + ``` + + You should get a prompt inside the pod. + +2. Install the ping command. + + ```bash + apt update && apt install iputils-ping + ``` + +3. Ping a known safe IP (like 8.8.8.8, Google’s public DNS server). + + ```bash + ping 8.8.8.8 + ``` + +4. Open the [FEODO tracker list](https://feodotracker.abuse.ch/downloads/ipblocklist.txt) and choose an IP on the list to ping. + You should not get connectivity, and the pings will show up as denied traffic in the flow logs. + +## Additional resources + +See [GlobalThreatFeed](../reference/resources/globalthreatfeed.mdx) resource definition for all configuration options. diff --git a/calico-cloud_versioned_docs/version-24-1/threat/tor-vpn-feed-and-dashboard.mdx b/calico-cloud_versioned_docs/version-24-1/threat/tor-vpn-feed-and-dashboard.mdx new file mode 100644 index 0000000000..3fd4dcfd87 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/tor-vpn-feed-and-dashboard.mdx @@ -0,0 +1,65 @@ +--- +description: Detect anonymization activity in Calico Cloud connected clusters with Tor bulk exit and X4B VPN feeds, and investigate findings in the Tor-VPN dashboard in the web console. +--- + +# Anonymization attacks + +## Big picture + +Detect and analyze malicious anonymization activity using Tor-VPN feeds. + +## Value + +**Tor and VPN infrastructure** are used in enabling anonymous communication, where an attacker can leverage anonymity to scan, attack or compromise the target. It’s hard for network security teams to track malicious actors using such anonymization tools. Hence **Tor and VPN feeds** come into play where the feeds track all the Tor bulk exit nodes as well as most of the anonymizing VPN infrastructure on the internet. **The Tor-VPN Dashboard** helps network security teams to monitor and respond to any detected activity where they have a clusterwide view and granular control over logs which is critical in stopping the possible attack in early stages. + +## Concepts + +### About Tor and VPN threats + +**Tor** is a popular anonymization network on the internet. It is also popular among the malicious actors, hacktivist groups, criminal enterprises as the infrastructure hides the real identity of an attacker carrying out malicious activities. To track down such attackers, Tor historically was subject to investigation by various state level intelligence agencies from US and UK for criminal activities such as Silk Road marketplace, Mirai Botnet C&C. Though it’s not possible to completely de-anonymize the attacker. Hence **Tor bulk exit feed** came into existence to track all the Tor exit IPs over the internet to know attackers using the Tor infrastructure. +Over the years, many Tor flaws became public and attackers evolved to leverage Tor network with additional VPN layers. There are many individual VPN providers which have the anonymizing infrastructure. Attackers can use these new breed of VPN providers with existing options like Tor to make sure of anonymity. To help security teams, the **X4B vpn feed** detects all the major VPN providers on the internet. + +### Tor-VPN feed types + +**Tor Bulk Exit feed** +The Tor Bulk Exit feed lists available Tor exit nodes on the internet which are used by Tor network. The list is continuously updated and maintained by the Tor project. An attacker using Tor network, is likely to use one of the bulk exit nodes to connect to your infrastructure. The network security teams can detect such activity with Tor bulk exit feed and investigate as required. + +**X4B VPN feed** +In recent times it became a trend to use multiple anonymization networks to hide real attacker identity. There are lots of lists of open proxies and tor nodes on the web, but surprisingly few usable ones dedicated to VPN providers and datacenters. This list combines known VPN netblocks and ASNs owned by datacenters and VPN providers. This list doesn't list all VPNs, but should list the vast majority of common ones. + +### The $[prodname] Tor-VPN dashboard + +The Tor-VPN dashboard helps network security teams to monitor and respond to any detected activity by Tor and VPN feeds. It provides a cluster context to the detection and shows multiple artifacts e.g. flow logs, filtering controls, a tag cloud and line graph to analyze the activity and respond faster. + +## Before you begin... + +### Required + +Privileges to manage GlobalThreatFeed i.e. clusterrole `intrusion-detection-controller` + +### Recommended + +We recommend that you turn down the aggregation of flow logs sent to Elasticsearch for configuring threat feeds. If you do not adjust flow logs, $[prodname] aggregates over the external IP addresses for allowed traffic, and threat feed searches will not provide useful results (unless the traffic is denied by policy). Go to: [FelixConfiguration](../reference/resources/felixconfig.mdx) and set the field, **flowLogsFileAggregationKindForAllowed** to **1**. + +## How to + +In this section we will look at how to add Tor and VPN feeds to $[prodname]. Installation process is straightforward as below. + +1. Add threat feed to the cluster. + For VPN Feed, + ```shell + kubectl apply -f $[filesUrl_CE]/manifests/threatdef/vpn-feed.yaml + ``` + For Tor Bulk Exit Feed, + ```shell + kubectl apply -f $[filesUrl_CE]/manifests/threatdef/tor-exit-feed.yaml + ``` +2. Feeds can be checked using following command: + ```shell + kubectl get globalthreatfeeds + ``` + +## Additional resources + +- See [GlobalThreatFeed](../reference/resources/globalthreatfeed.mdx) resource definition for all configuration options. +- Check example to Trace and block Suspicious IPs [Here](suspicious-ips.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/threat/web-application-firewall.mdx b/calico-cloud_versioned_docs/version-24-1/threat/web-application-firewall.mdx new file mode 100644 index 0000000000..6d6d841c77 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/threat/web-application-firewall.mdx @@ -0,0 +1,384 @@ +--- +description: Protect cluster workloads from Layer 7 attacks with the Calico Cloud workload-based WAF, powered by Envoy sidecars and the OWASP ModSecurity Core Rule Set. +--- + +# Workload-based Web Application Firewall (WAF) + +:::note + +This feature is tech preview. Tech preview features may be subject to significant changes before they become GA. + +::: + +## Big picture + +Protect cloud-native applications from application layer attacks with $[prodname] Workload-based Web Application Firewall (WAF). + +## Value + +Our workload-centric Web Application Firewall (WAF) protects your workloads from a variety of application layer attacks originating from within your cluster such as [SQL injection](https://cheatsheetseries.owasp.org/cheatsheets/SQL_Injection_Prevention_Cheat_Sheet.html). Given that attacks on apps are the [leading cause of breaches](https://www.f5.com/labs/articles/threat-intelligence/application-protection-report-2019--episode-2--2018-breach-trend), you need to secure the HTTP traffic inside your cluster. + +Historically, web application firewalls (WAFs) were deployed at the edge of your cluster to filter incoming traffic. Our workload-based WAF solution takes a unique, cloud-native approach to web security by allowing you to implement zero-trust rules for workloads inside your cluster. + +## Concepts + +### About $[prodname] WAF + +WAF is deployed in your cluster with Envoy as a sidecar container on your pods. $[prodname] proxies traffic through Envoy, checking HTTP requests using the industry-standard +[ModSecurity](https://coreruleset.org/) with OWASP Core Rule Set `v4.7.0` with some modifications for Kubernetes workloads. + +You simply enable WAF in the web console, and determine the deployments that you want to enable for WAF protection. By default WAF is set to `DetectionOnly` so no traffic will be denied until you are ready to turn on blocking mode. + +Every request that WAF finds an issue with, will result in a Security Event being created for [you to review in the UI](#view-waf-events), regardless of whether the traffic was allowed or denied. This can greatly help in tuning later. + +#### How WAF determines if a request should be allowed or denied + +If you configure WAF in blocking mode, WAF will use something called [anomaly scoring mode](https://coreruleset.org/docs/2-how-crs-works/2-1-anomaly_scoring/) to determine if a request is allowed with `200 OK` or denied `403 Forbidden`. + +This works by matching a single HTTP request against all the configured WAF rules. Each rule has a score and WAF adds all the matched rule scores together, and compares it to the overall anomaly threshold score (100 by default). If the score is under the threshold the request is allowed and if the score is over the threshold the request is denied. Our WAF starts in detection mode only and with a high default scoring threshold so is safe to turn on and then [fine-tune the WAF](#manage-waf-configuration) for your specific needs in your cluster. + +## Before you begin + +### Limitations + +WAF cannot be used with: +* Host-networked client pods +* TLS traffic +* [LoadBalancer services](https://Kubernetes.io/docs/concepts/services-networking/service/#loadbalancer) +* Kubernetes 1.28 or earlier + +:::note + +When selecting and deselecting traffic for WAF, active connections may be disrupted. + +::: + +:::caution + +You should enable WAF only for your application pods. +If you enable WAF for deployments that are part of the Kubernetes system itself, you may create an undesirable cluster state. + +::: + +## Enable WAF + +### Enable WAF using the CLI + +#### Using kubectl + +Create an (or edit an existing) ApplicationLayer resource named `tigera-secure`, set the `sidecarInjection` field to `Enabled`. + +```bash +kubectl apply -f - < Web Application Firewall > Configure Web Application Firewall**. + + WAF services + +Then, wait for the Tigera APIServer deployment and Log collection daemonset to progress to ready. + +```bash +kubectl rollout status -n calico-system deployment/calico-apiserver --timeout=120s +kubectl rollout status -n calico-system ds/l7-log-collector --timeout=120s +``` + +### (Optional) Deploy a sample application + +If you don’t have an application to test WAF with or don’t want to use it right away against your own application, +we recommend that you install the [GoogleCloudPlatform/microservices-demo app](https://github.com/GoogleCloudPlatform/microservices-demo): + +```bash +kubectl apply -f https://raw.githubusercontent.com/GoogleCloudPlatform/microservices-demo/main/release/kubernetes-manifests.yaml +``` + +## Apply WAF to deployments + +Now that you have deployed WAF in your cluster, you can select the deployments you want to protect from application layer attacks. + +If you have deployed the sample application, you can apply WAF on a deployment associated with your app, as follows: + +```bash +kubectl patch deployment frontend -n default -p '{"spec":{"template":{"metadata":{"labels":{"applicationlayer.projectcalico.org/sidecar":"true"},"annotations":{"applicationlayer.projectcalico.org/waf":"Enabled"}}}}}' +``` + +Alternatively, you can use the web console to apply WAF to the `frontend` deployment if you opted for sidecars. + +In this example, we applied WAF to the `frontend` pods. This means that every request that goes through the `frontend` deployment is inspected. +However, the traffic is not blocked because the WAF rule is set to `DetectionOnly` by default. You can adjust rules and start blocking traffic by [fine-tuning your WAF](#manage-waf-configuration). + +In the previous example, we applied WAF to the `frontend` deployment of the sample application. Here, we are +applying WAF to a deployment of your own application. + +1. On the web console, click **Threat Defense**, **Web Application Firewall**. +2. Select the deployments you want WAF to inspect, and then click **Confirm Selections**. + + WAF services + +3. On the **Web Application Firewall** page, you can verify that WAF is enabled for a deployment by locating the deployment and checking that the **Status** column says **Enabled**. + +4. To make further changes to a deployment, click **Actions**, and then **Enable** or **Disable**. + +You have now applied WAF rule sets to your own deployments, and note that the traffic that goes through the selected deployments will be alerted but not blocked by default. + +### Trigger a WAF event + +If you would like to trigger a WAF event for testing purposes, you can simulate an SQL injection attack inside your cluster by crafting a HTTP request with a query string that WAF will detect as an SQL injection attempt. +The query string in this example has some SQL syntax embedded in the text. This is harmless and for demo purposes, but WAF will detect this pattern and create an event for this HTTP request. + +Run a simple curl command from any pod inside your cluster targeting a deployment you have selected for WAF protection e.g. from the demo app above we could attempt to send a simple HTTP request to the cartservice. + +``` +curl http://frontend/cart?artist=0+div+1+union%23foo*%2F*bar%0D%0Aselect%23foo%0D%0A1%2C2%2Ccurrent_user +``` + +## View default rule set details from the web console + +You can view the default rules from the Core Rule Set (OWASP Top 10) by using the web console. +Viewing the rules gives you an overview of the rule set, and it can be used when customizing your WAF configuration. +This method always shows you the default configuration, even if you have added customizations. + +1. From the web console, go to **Threat > Web Application Firewall**, and then click the **Rulesets** tab. +1. Click the rule set card. +1. To view details about specific rules, click any item in the list. + +## Manage WAF configuration + +Reviewing the default rule set config: + +```bash +Include coraza.conf +Include crs-setup.conf +Include coreruleset/*.conf + +SecRuleEngine DetectionOnly +``` + +The configuration file starts with importing the appropriate rule set config. We use Coraza WAF's recommended [Core Rule Set setup](https://coraza.io/docs/tutorials/coreruleset/) files: + +1. Coraza recommended configuration with [coraza.conf](https://github.com/corazawaf/coraza-coreruleset/blob/main/rules/%40coraza.conf-recommended) and [crs-setup.conf](https://github.com/corazawaf/coraza-coreruleset/blob/main/rules/%40crs-setup.conf.example) (kept in the same ConfigMap) +1. The rest of the [coreruleset](https://github.com/corazawaf/coraza-coreruleset/tree/main/rules/%40owasp_crs) files, currently [v4.7.0](https://github.com/corazawaf/coraza-coreruleset/tree/v4.7.0/rules/%40owasp_crs) + +These files can be customized if desired. Add all your customizations directly under `tigera.conf`: + +```bash +kubectl edit cm -n tigera-operator tigera-waf-config +``` + +After editing this ConfigMap successfully, the `tigera-waf-config` ConfigMap will be replaced in the `tigera-operator` namespace, +which then triggers a rolling restart of your L7 pods. This means that the HTTP connections going through L7 pods at the time of pod termination will be (RST) reset. + +:::note + +It is important to adhere to the [Core Rule Set documentation](https://coreruleset.org/docs) on how to edit the behaviour of + your WAF. A good place to begin at is the [Installing Core Rule Set](https://coreruleset.org/docs/1-getting-started/1-1-crs-installation/). + +In many scenarios, the default example CRS configuration will be a good enough starting point. It is recommended to review the example configuration file before +you deploy it to make sure it’s right for your environment. +::: + +### Customization options + +#### Set WAF to block traffic +By default WAF will not block a request even if it has matching rule violations. The rule engine is set to `DetectionOnly`. You can configure to block traffic instead with an `HTTP 403 Forbidden` response status code when the combined matched rules scores exceed a certain threshold. + +1. Edit the configmap: + ```bash + kubectl edit cm -n tigera-operator tigera-waf-config + ``` +2. Look for `SecRuleEngine DetectionOnly` and change it to `SecRuleEngine On`. +3. Save your changes. This triggers a rolling update of the L7 pods. + +| Action | Description | Disruptive? | +| ------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- | +| DetectionOnly | Traffic is not denied nor dropped. $[prodname] will log events. | No +| On | Denies HTTP traffic. $[prodname] will log the event in Security Events. | Yes | +| Off | Be cautious about using this option. Traffic is not denied, and there are no events. |No | Yes | + +#### Other basic customizations + +For basic customizations, it's best to add it after all the includes in `tigera.conf`. In fact, this is the reason why the `SecRuleEngine` directive and the rest of [our customizations](https://github.com/tigera/operator/blob/master/pkg/render/applicationlayer/ruleset/coreruleset/tigera.conf#L8-L17) are situated there. + +An example is adding a sampling mode. For that, the `tigera.conf` will look like this: + +```bash +# CoreRuleset activation +Include coraza.conf +Include crs-setup.conf +Include coreruleset/*.conf + +SecRuleEngine DetectionOnly + +# --- all customizations appear below this line, unless they need a specific loading order like plugins --- + +# --- Add sampling mode +# Read about sampling mode here https://coreruleset.org/docs/2-how-crs-works/2-4-sampling_mode/ +SecAction "id:900400,\ + phase:1,\ + pass,\ + nolog,\ + setvar:tx.sampling_percentage=50" +``` + +Also you can disable certain rules here: + +```bash +# --- disable 'Request content type is not allowed by policy' +SecRuleRemoveById 920420 +``` + +Change anomaly scoring threshold: + +```bash + SecAction \ + "id:900110,\ + phase:1,\ + nolog,\ + pass,\ + t:none,\ + setvar:tx.inbound_anomaly_score_threshold=25,\ + setvar:tx.outbound_anomaly_score_threshold=20" +``` + +Or even change rule action parameters or behavior. For example: + + ```bash + # --- append to more allowed content types to request bodies +SecAction \ + "id:900220,\ + phase:1,\ + nolog,\ + pass,\ + t:none,\ + setvar:'tx.allowed_request_content_type=|application/x-www-form-urlencoded| |multipart/form-data| |multipart/related| |text/xml| |application/xml| |application/soap+xml| |application/json| |application/cloudevents+json| |application/cloudevents-batch+json| |application/grpc|'" +``` + +#### Using Core Rule Set plugins + +Let's go with an example plugin: [Wordpress Rule Exclusions](https://github.com/coreruleset/wordpress-rule-exclusions-plugin/). + +Plugin files are the following: + +``` +wordpress-rule-exclusions-before.conf +wordpress-rule-exclusions-config.conf +``` + +To include these files properly, structure your work directory like so: + +``` +tigera.conf +coraza.conf +crs-setup.conf +wordpress-rule-exclusions-before.conf +wordpress-rule-exclusions-config.conf +``` + +and then `tigera.conf` contents should be: + +```bash +Include coraza.conf + +Include wordpress-rule-exclusions-config.conf +Include wordpress-rule-exclusions-before.conf + +Include crs-setup.conf +Include coreruleset/*.conf + +# if your plugin has an -after.conf, include them here +# but wordpress rule exclusions doesn't so we're adding a comment placeholder +# Include wordpress-rule-exclusions-after.conf + +SecRuleEngine DetectionOnly +``` + +Then create and apply the configmap: + +```bash +## create the configuration map itself +kubectl create cm --dry-run=client \ + --from-file=tigera.conf \ + --from-file=coraza.conf \ + --from-file=crs-setup.conf \ + --from-file=wordpress-rule-exclusions-config.conf \ + --from-file=wordpress-rule-exclusions-before.conf \ + -n tigera-operator tigera-waf-config -o yaml > ruleset.configmap.yaml + +## replace active configmap +kubectl replace -f ruleset.configmap.yaml +``` + +Read more about the order of execution for plugins here: https://coreruleset.org/docs/4-about-plugins/4-1-plugins/ + +## View WAF events + +### Security Events + +To view WAF events in a centralized security events dashboard, go to: **Threat defense**, **Security Events**. For help, see [Security Event Management](../threat/security-event-management). + +### Disable WAF for a deployment + +To disable WAF on a deployment, use the Actions menu on the WAF board, or use the following command: + +```bash +kubectl patch deployment -n --type='json' -p '[{"op":"remove","path":"/spec/template/metadata/labels/applicationlayer.projectcalico.org~1sidecar"},{"op":"remove","path":"/spec/template/metadata/annotations/applicationlayer.projectcalico.org~1waf"}]' +``` + +## Disable WAF feature from your cluster + +To safely disable WAF, determine how to handle ApplicationLayer features and follow the steps: + +### Disable all ApplicationLayer features, including WAF + +```bash +kubectl delete applicationlayer tigera-secure +``` + +### Keep some ApplicationLayer features enabled + +To disable WAF but keep some ApplicationLayer features enabled, you must update the [ApplicationLayer](../reference/installation/api#applicationlayer) custom resource. + +Note that the [ApplicationLayer Specification](../reference/installation/api#applicationlayerspec) can specify configuration for [application logging](../reference/installation/api#operator.tigera.io/v1.logcollectionspec) and [application layer policy](../reference/installation/api#operator.tigera.io/v1.applicationlayerpolicystatustype) also. + +For the ApplicationLayer custom resource to be valid, at least one of these features have to be enabled, for example: + + +#### Valid YAML + +WAF disabled, ALP enabled (using per-host proxy), and Log collection is unspecified (and the default is disabled) + +```yaml +apiVersion: operator.tigera.io/v1 +kind: ApplicationLayer +metadata: + name: tigera-secure +spec: + webApplicationFirewall: Disabled + applicationLayerPolicy: Enabled + sidecarInjection: Disabled +``` + +##### Invalid YAML + +WAF and ALP both set as disabled, log collection is unspecified (and the default is disabled) +```yaml +apiVersion: operator.tigera.io/v1 +kind: ApplicationLayer +metadata: + name: tigera-secure +spec: + webApplicationFirewall: Disabled + applicationLayerPolicy: Disabled + sidecarInjection: Disabled +``` diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/applications/egress-controls.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/applications/egress-controls.mdx new file mode 100644 index 0000000000..d32ac684a4 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/applications/egress-controls.mdx @@ -0,0 +1,206 @@ +--- +description: Step-by-step tutorial for restricting application egress in Calico Cloud using network sets, domain wildcards, and global network sets with network policy. +--- + +# Secure egress access from workloads to destinations outside the cluster + +In this article we'll show you how to restrict egress access for your application or microservice pods to external endpoints outside of the cluster. + +- [Use a network set with a network policy](#use-a-network-set-with-a-network-policy) +- [Use wildcards in domain names](#use-wildcards-in-domain-names) +- [Use a global network set with a global network policy](#use-a-global-network-set-with-a-global-network-policy) + +## What are networks sets? + +$[prodname] NetworkSet and GlobalNetworkSet resources are used to define endpoints external to the Kubernetes cluster. The scope of a NetworkSet is the **namespace where they are defined**; the scope of a GlobalNetworkSet is **cluster-wide**. + +NetworkSet/GlobalNetworkSet come in two types: + +- **Network-based** - defines IP address-based external endpoints +- **Domain-based** - defines URL-based external endpoints. Using domains in policy is often called, DNS policy. + +## Use a network set with a network policy + +In this example, we have a microservice that requires egress access to two external endpoints: a repo and a partner. + +![egress-example](/img/calico-cloud/egress-example.png) + +`svc3` pods needs egress access to: + +- A repo named, `app2-repo` at domain `app2-repo.example.com`, port 443 +- A partner named, `app2-partners` at endpoint `10.10.10.10/32`, port 1010 and 53 + +First, we define a domain-based NetworkSet. Using `allowedEgressDomains` we can specify the trusted repo by its URL, `app2-repo.example.com`. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkSet +metadata: + name: app2-repo + namespace: app2-ns + labels: + trusted-ep: 'app2-repo' +spec: + allowedEgressDomains: + - 'app2-repo.example.com' +``` + +Next, we create a network-based NetworkSet that specifies the IP address of the trusted partner, `10.10.10.10/32`. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkSet +metadata: + name: app2-partners + namespace: app2-ns + labels: + trusted-ep: 'app2-partners' +spec: + nets: + - 10.10.10.10/32 +``` + +Now we can reference these NetworkSets by their labels in NetworkPolicy. We use a selector to specify the service, `app == "app2"&&svc == "svc3"`, and then selectors to allow egress to our two trusted endpoints, `selector: trusted-ep == "app2-repo"` at port 443, and `selector: trusted-ep == "app2-partners"` at ports 1010 and 53. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: application.app2-svc3-egress + namespace: app2-ns +spec: + tier: application + selector: (app == "app2" && svc == "svc3") + egress: + - action: Allow + source: {} + destination: + selector: trusted-ep == "app2-repo" + ports: + - '443' + protocol: TCP + - action: Allow + source: {} + destination: + selector: trusted-ep == "app2-partners" + ports: + - '1010' + protocol: TCP + - action: Allow + protocol: UDP + source: {} + destination: + ports: + - '53' + types: + - Egress +``` + +## Use wildcards in domain names + +In this example, we create another namespaced NetworkPolicy with egress rules with `action: Allow` and a `destination.domains` field specifying the domain names to which egress traffic is allowed. + +The first egress rule allows DNS traffic using UDP over port 53, and the second rule allows connections outside the cluster to domains `api.alice.com` and `*.example.com` (which means `.example.com`, such as `bob.example.com`). + +Note that our namespaced NetworkPolicy can only grant egress access to specified domains, and to workload endpoints in the `rollout-test` namespace. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: allow-egress-to-domains + namespace: rollout-test +spec: + order: 1 + selector: my-pod-label == 'my-value' + types: + - Egress + egress: + - action: Allow + protocol: UDP + destination: + ports: + - 53 + - dns + - action: Allow + destination: + domains: + - api.alice.com + - '*.example.com' +``` + +## Use a global network set with a global network policy + +We recommend using a GlobalNetworkSet when the same set of domains needs to be referenced in multiple policies, or when you want the allowed destinations to be a mix of domains and IPs from global network sets, or IPs from workload endpoints and host endpoints. By using a single destination selector in a global network set, you can potentially match all of these resources. + +In the following example, the allowed egress domains (`api.alice.com` and `*.example.com`) are specified in the GlobalNetworkSet. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkSet +metadata: + name: allowed-domains-1 + labels: + color: red +spec: + allowedEgressDomains: + - api.alice.com + - '*.example.com' +``` + +Then, we reference the global network set in a GlobalNetworkPolicy using a destination label selector. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: allow-egress-to-domain +spec: + order: 1 + selector: my-pod-label == 'my-value' + types: + - Egress + egress: + - action: Allow + destination: + selector: color == 'red' +``` + +## For reference... + +### Allowed egress domains + +Using domain names in policy rules is limited to only egress allow rules. $[prodname] allows connections only to IP addresses returned from DNS lookups to trusted DNS servers. The supported DNS types are: A, AAAA, and CNAME records. The domain name must be an exact match; for example, **google.com** is treated as distinct from **www.google.com**. + +**Note:** Kubernetes labels provide a similar convenience for services within the cluster. $[prodname] does not support using domain names for services within the cluster. Use Kubernetes labels for services within the cluster. + +### Domain name matching + +When a configured domain name has no wildcard (`*`), it matches exactly that domain name. For example: + +- `microsoft.com` +- `tigera.io` + +With a single asterisk in any part of the domain name, it matches 1 or more path components at that position. For example: + +- `*.google.com` matches `www.google.com` and `www.ipv6.google.com`, but not `google.com` +- `www.*.com` matches `www.sun.com` and `www.apple.com`, but not `www.com` +- `update.*.mycompany.com` matches `update.tools.mycompany.com`, `update.secure.suite.mycompany.com`, and so on + +**Not** supported are: + +- Multiple wildcards in the same domain, for example: `*.*.mycompany.com` +- Asterisks that are not the entire component, for example: `www.g*.com` +- More general wildcards, such as regular expressions + +### Workload and host endpoints + +Policy with domain names can be enforced on workload or host endpoints. When a policy with domain names applies to a workload endpoint, it allows that workload to connect out to the specified domains. When policy with domain names applies to a host endpoint, it allows clients directly on the relevant host (including any host-networked workloads) to connect out to the specified domains. + +### Trusted DNS servers + +$[prodname] trusts DNS information only from its list of DNS trusted servers. Using trusted DNS servers to back domain names in policy, prevents a malicious workload from using IPs returned by a fake DNS server to hijack domain names in policy rules. + +By default, $[prodname] trusts the Kubernetes cluster’s DNS service (kube-dns or CoreDNS). For workload endpoints, these out-of-the-box defaults work with standard Kubernetes installs, so normally you won’t change them. For host endpoints you will need to add the IP addresses that the cluster nodes use for DNS resolution. + +To change the default DNS trusted servers, use the DNSTrustedServers parameter in Felix. diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/applications/index.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/applications/index.mdx new file mode 100644 index 0000000000..034b18d926 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/applications/index.mdx @@ -0,0 +1,11 @@ +--- +description: Index of Calico Cloud tutorials that secure ingress and egress access for applications and microservices using network policy, network sets, and DNS rules. +hide_table_of_contents: true +--- + +# Secure ingress and egress for applications + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/applications/ingress-microservices.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/applications/ingress-microservices.mdx new file mode 100644 index 0000000000..76b0d8e113 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/applications/ingress-microservices.mdx @@ -0,0 +1,275 @@ +--- +description: Step-by-step tutorial for writing Calico Cloud network policies that grant ingress access to a microservice from internal clients, services, and load balancers. +--- + +# Secure ingress access to a microservice or application + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +In this article you will learn how to create $[prodname] policies that securely allow ingress access to a microservice or application. + + + + +## Secure ingress access to a microservice + +In this example, we have a microservices with four services (sv1, sv2, sv3, and sv4). Let's assume the following requirements: + +- Everyone internal to the organization can access `svc1` pods at TCP port 10001 +- `svc1` pods can access `svc2` at TCP port 10002 +- `svc3` pods can access at TCP port 10003 +- `svc2` pods can access `vc4` at TCP ports 10004, 10005 + +Also, because `svc1` pods pass through a trusted load balancer, we want to allow ingress traffic for the load balancer. + +![ingress-microservices](/img/calico-cloud/ingress-microservices.png) + +Let's start with securing ingress access to the trusted load balancer. We will use the $[prodname] **GlobalNetworkSet** resource with cluster-wide scope to define the load balancer IP addresses; this allows the same trusted external endpoints to be used across multiple namespaces. + +**GlobalNetworkSet for load balancer** + +This GlobalNetworkSet contains the IP addresses for the trusted load balancer. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkSet +metadata: + name: load-balancer + labels: + trusted-ep: "load-balancer" +spec: + nets: + # Modify the ip addresses to refer to the ip addresses of load-balancers in your environment + - 10.0.0.1/32 + - 10.0.0.2/32 +``` + +Next, we will create four network policies. + +**NetworkPolicy 1** + +This policy allows ingress from the trusted load balancer to pods in `sv1`. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: application.app2-svc1 + namespace: app2-ns +spec: + tier: application + order: 500 + selector: (app == "app2" && svc == "svc1") + ingress: + - action: Allow + protocol: TCP + source: + selector: trusted-ep == "load-balancer" + destination: + ports: + - '10001' + types: + - Ingress +``` + +**NetworkPolicy 2** + +This policy allows ingress access from `sv1` pods to `sv2` pods. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: application.app2-svc2 + namespace: app2-ns +spec: + tier: application + order: 600 + selector: (app == "app2" && svc == "svc2") + ingress: + - action: Allow + protocol: TCP + source: + selector: svc == "svc1" + destination: + ports: + - '10002' + types: + - Ingress +``` + +**NetworkPolicy 3** + +This policy allows ingress access from `svc1` pods to `svc3` pods. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: application.app2-svc3 + namespace: app2-ns +spec: + tier: application + order: 700 + selector: (app == "app2" && svc == "svc3") + ingress: + - action: Allow + protocol: TCP + source: + selector: svc == "svc1" + destination: + ports: + - '10003' + types: + - Ingress +``` + +**NetworkPolicy 4** + +This policy allows ingress access from `svc2` pods to `svc4` pods. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: application.app2-svc4 + namespace: app2-ns +spec: + tier: application + order: 800 + selector: (app == "app2" && svc == "svc4") + ingress: + - action: Allow + protocol: TCP + source: + selector: svc == "svc2" + destination: + ports: + - '10003' + - '10004' + types: + - Ingress +``` + + + + +## Secure ingress access to an application + +In this example, we have an application with frontend, backend, and a database. Let's assume the following requirements: + +- Everyone internal to the organization can access the `frontend` at TCP port 10001 +- `frontend` can access the `backend` at TCP port 10002 +- `backend` can access the `database` at TCP ports 10003, 10004 + +Also, because frontend pods pass through a trusted load balancer, we want to allow ingress traffic for the load balancer. + +![ingress-application](/img/calico-cloud/ingress-application.png) + +Let's start with securing ingress access to the trusted load balancer. We will use the $[prodname] **GlobalNetworkSet** resource with cluster-wide scope to define the load balancer IP addresses; this allows the same trusted external endpoints to be used across multiple namespaces. + +**GlobalNetworkSet for load balancer** + +This GlobalNetworkSet contains the IP addresses for the trusted load balancer. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkSet +metadata: + name: load-balancer + labels: + trusted-ep: "load-balancer" +spec: + nets: + # Modify the ip addresses to refer to the ip addresses of load-balancers in your environment + - 10.0.0.1/32 + - 10.0.0.2/32 +``` + +Next, we will create three network policies. + +**NetworkPolicy 1** + +This policy allows ingress from the trusted load balancer to pods in `frontend`. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: application.app1-frontend + namespace: app1-ns +spec: + tier: application + order: 200 + selector: (app == "app1" && svc == "frontend") + serviceAccountSelector: '' + ingress: + - action: Allow + protocol: TCP + source: + selector: trusted-ep == "load-balancer" + destination: + ports: + - '10001' + types: + - Ingress +``` + +**NetworkPolicy 2** + +This policy allows ingress access from pods in the `frontend` to pods in the `backend`. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: application.app1-backend + namespace: app1-ns +spec: + tier: application + order: 300 + selector: (app == "app1" && svc == "backend") + serviceAccountSelector: '' + ingress: + - action: Allow + protocol: TCP + source: + selector: svc == "frontend" + destination: + ports: + - '10002' + types: + - Ingress +``` + +**NetworkPolicy 3** + +This policy allows ingress access from pods in the `backend` to pods in the `database`. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: application.app1-database + namespace: app1-ns +spec: + tier: application + order: 400 + selector: (app == "app1" && svc == "database") + serviceAccountSelector: '' + ingress: + - action: Allow + protocol: TCP + source: + selector: svc == "backend" + destination: + ports: + - '10003' + - '10004' + types: + - Ingress +``` + + + diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/calico-cloud-features/index.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/calico-cloud-features/index.mdx new file mode 100644 index 0000000000..4507859312 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/calico-cloud-features/index.mdx @@ -0,0 +1,11 @@ +--- +description: Index of Calico Cloud feature tutorials covering the web console tour, Service Graph, network sets, and projects. +hide_table_of_contents: true +--- + +# Web console features + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/calico-cloud-features/networksets.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/calico-cloud-features/networksets.mdx new file mode 100644 index 0000000000..1aed8972f4 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/calico-cloud-features/networksets.mdx @@ -0,0 +1,260 @@ +--- +description: Step-by-step tutorial for using network sets and global network sets in Calico Cloud to model external endpoints and reuse IP and domain lists in policy. +--- + +# Understanding network sets + +## Visualize traffic to/from your cluster + +Modern applications often integrate with third-party APIs and SaaS services that live outside Kubernetes clusters. To securely enable access to those integrations, you must be able to limit IP ranges for egress and ingress traffic to workloads. Limiting IP lists or ranges is also used to deny-list bad actors or embargoed countries. To limit IP ranges, you need to use the $[prodname] resource called **network sets**. + +## What are network sets? + +**Network sets** are a grouping mechanism that allows you to create an arbitrary set of IP subnetworks/CIDRs or domains that can be matched by standard label selectors in Kubernetes or $[prodname] network policy. Like IP pools for pods, they allow you to reuse/scale sets of IP addresses in policies. + +A **network set** is a namespaced resource that you can use with Kubernetes or $[prodname] network policies; a **global network set** is a cluster-wide resource that you can use with $[prodname] network policies. + +Like network policy, you manage user access to network sets using standard Kubernetes RBAC. + +## Why are network sets powerful? + +If you are familiar with Service Graph in the web console, you know the value of seeing pod-to-pod traffic within your cluster. But what about traffic external to your cluster? + +$[prodname] automatically detects IPs for pods and nodes that fall into the standard IETF “public network” and “private network” designations, and displays those as icons in Service Graph. So you get some visibility into external traffic without using any network sets. + +![public-private-networks](/img/calico-cloud/public-private-networks.png) + +However, when you create network sets, you can get more granular visibility into what's leaving the cluster to public networks. Because you control the grouping, the naming, and labeling, you create visibility that is customized to your organization. This is why they are so powerful. + +Here are just a few examples of how network sets can be used: + +- **Egress access control** + + Network sets are a key resource for defining egress access controls; for example, securing ingress to microservices/apps or egress from workloads outside the cluster. + +- **Troubleshooting** + + Network sets appear as additional metadata in flow logs, Flow Visualizer, and Service Graph. + +- **Efficiency and scaling** + + Network sets are critical when scaling your deployment. You may have only a few CIDRs when you start. But as you scale out, it is easier to update a handful of network sets than update each network policy individually. Also, in a Kubernetes deployment, putting lots of anything (CIDRs, ports, policy rules) directly into policies causes inefficiencies in traffic processing (iptables/eBPF). + +- **Microsegmentation and shift left** + + Network sets provide the same microsegmentation controls as network policy. For example, you can allow specific users to create policies (that reference network sets), but allow only certain users to manage network sets. + +- **Threat defense** + + Network sets are key to being able to manage threats by blocking bad IPs with policy in a timely way. Imagine having to update individual policies when you find a bad IP you need to quickly block. You can even give access to a controller that automatically updates CIDRs in a network set when a bad IP is found. + +## Create a network set and use it in policy + +In this section, we’ll walk through how to create a namespaced network set in the web console. You can follow along using your cluster or tigera-labs cluster. + +In this example, you will create a network set named, `google`. This network set contains a list of trusted google endpoints for a microservice called, `hipstershop`. As a service owner, you want to be able to see traffic leaving the microservices in Service Graph. Instead of matching endpoints on IP addresses, we will use domain names. + +1. From the left navbar, click **Network Sets**. +1. Click **Add Network Set**, and enter these values. + - For Name: `google` + - For Scope: Select **Namespace** and select, `hipstershop` +1. Under Labels, click **Add label**. + - In the Select key field, enter `destinations` and click the green bar to add this new entry. + - In the Value field, enter `google`, click the green bar to add the entry, and save. +1. For Domains, click **+Add Domain** and these URLs: `clouddebugger.googleapis.com`, `cloudtrace.googleapis.com`, `metadata.google.internal`, `monitoring.googleapis.com`. +1. Click **Create Network Set**. + +You’ve created your first network set. + +![add-networkset-google](/img/calico-cloud/add-networkset-google.png) + +The YAML looks like this: + +```yaml +kind: NetworkSet +apiVersion: projectcalico.org/v3 +metadata: + name: google + labels: + destinations: google + namespace: hipstershop +spec: + nets: [] + allowedEgressDomains: + - clouddebugger.googleapis.com + - cloudtrace.googleapis.com + - metadata.google.internal + - monitoring.googleapis.com +``` + +Next, we write a DNS policy for hipstershop that allows egress traffic to the trusted google sites. The following network policy allows egress access for all destination selectors labeled, `google`. Note that putting domains in a network set referencing it in policy is the best practice. Also, note that using `selector: all()` should only be used if all pods in the namespace can access all of the domains in the network set; if not, you should create separate policies accordingly. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: application.allow-egress-domain + namespace: hipstershop +spec: + tier: application + order: 0 + selector: all() + serviceAccountSelector: '' + egress: + - action: Allow + source: {} + destination: + selector: destinations == "google" + types: + - Egress +``` + +## Network sets in Service Graph + +Continuing with our `hipstershop` example, if you go to Service Graph, you see hipstershop (highlighted in yellow). + +![hipstershop](/img/calico-cloud/hipstershop.png) + +If we double-click `hipstershop` to drill down, we now see the `google` network set icon (highlighted in yellow). We now have visibility to traffic external from google sites to hipstershop. (If you are using the tigera-labs cluster, note that the network set will not be displayed as shown below.) + +![google-networkset](/img/calico-cloud/google-networkset.png) + +Service Graph provides a view into how services are interconnected in a consumable view, along with easy access to flow logs. You can also see traffic associated with network sets in volumetric display with Flow Visualizer. + +## Tutorial + +In the following example, we create a global network set resource for a trusted load-balancer that can be used with microservices and applications. The label, `trusted-ep: load-balancer` is how this global network set can be referenced in policy. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkSet +metadata: + name: load-balancer + labels: + trusted-ep: "load-balancer" +spec: + nets: + # Modify the ip addresses to refer to the ip addresses of load-balancers in your environment + - 10.0.0.1/32 + - 10.0.0.2/32 +``` + +The following network policy uses the `selector: trusted-ep == "load balancer"` to reference the above GlobalNetworkSet. All applications in the `app2-ns` namespace, that match `app2` and `svc1` are allowed ingress traffic from the trusted load balance on port 1001. + +```yaml +apiVersion: projectcalico.org/v3 +kind: NetworkPolicy +metadata: + name: application.app2-svc1 + namespace: app2-ns +spec: + tier: application + order: 500 + selector: (app == "app2"&&svc == "svc1") + ingress: + - action: Allow + protocol: TCP + source: + selector: trusted-ep == "load-balancer" + destination: + ports: + - '10001' + types: + - Ingress +``` + +### Advanced policy rules with network sets + +When you combine $[prodname] policy rules with network sets, you have powerful ways to fine-tune. The following example combines network sets with specific rules in a global network policy to deny access more quickly. +We start by creating a $[prodname] GlobalNetworkSet that specifies a list of CIDR ranges we want to deny: 192.0.2.55/32 and 203.0.113.0/24. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkSet +metadata: + name: ip-protect + labels: + ip-deny-list: 'true' +spec: + nets: + - 192.0.2.55/32 + - 203.0.113.0/24 +``` + +Next, we create two $[prodname] GlobalNetworkPolicy resources. The first is a high "order" policy that allows traffic as a default for things that don’t match our second policy, which is low "order" and uses the GlobalNetworkSet label as a selector to deny ingress traffic (IP-deny-list in the previous step). In the label selector, we also include the term, `!has(projectcalico.org/namespace)`, which prevents this policy from matching pods or NetworkSets that also have this label. To more quickly enforce the denial of forwarded traffic to the host at the packet level, use the `doNotTrack` and `applyOnForward` options. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: forward-default-allow +spec: + selector: apply-ip-protect == 'true' + order: 1000 + doNotTrack: true + applyOnForward: true + types: + - Ingress + ingress: + - action: Allow +--- +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: ip-protect +spec: + selector: apply-ip-protect == 'true' + order: 0 + doNotTrack: true + applyOnForward: true + types: + - Ingress + ingress: + - action: Deny + source: + selector: ip-deny-list == 'true' && !has(projectcalico.org/namespace) +``` + +## Best practices for using network sets + +- Create network sets as soon as possible after getting started + + This allows you to quickly realize the benefits of seeing custom metadata in flow logs and visualizing traffic in Service Graph and Flow Visualizer. + +- Create a network set label and name schema + + It is helpful to think: what names would be meaningful and easy to understand when you look in Service Graph or Flow Visualizer? What labels will be easy to understand when used in network policies – especially if you are separating users who manage network sets from those who consume them in network policies. + +- Do not put large sets of CIDRs and domains directly in policy + + Network sets allow you to specify CIDRs and/or domains. Although you can add CIDRs and domains directly in policy, it doesn't scale. + +- Do not put thousands of rules into a policy, each with a different CIDR + + If your set of /32s can be easily aggregated into a few broader CIDRs without compromising security, it’s a good thing to do; whether you’re putting the CIDRs in the rule or using a network set. + +- If you want to match thousands of endpoints, write one or two rules and use selectors to match the endpoints. + + Having one rule per port, per host is inefficient because each rule ends up being rendered as an iptables/eBPF rule instead of making good use of IP sets. + +- Avoid overlapping IP addresses/subnets in networkset/globalnetworkset definitions + +The following table provides guidance on the efficient use of network sets. + +| Policy | Network set | Results | +| ------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------- | --------------------------------------------------------------------------------------------------------------------- | +| source: selector: foo="bar" | With handful of broad CIDRs | **Efficient**
**-** 1 iptables/eBPF rule
- 1 IP set with handful of CIDRs | +| source: nets: [ ... handful ...] | Not used | **Efficient**
- Handful of iptables/eBPF rules
- 0 IP sets | +| source: selector: foo="bar" | One network set with 2000 x /32s | **Fairly efficient**
- 1 iptables/eBPF rule
- 1 IP sets with 2000 entries | +| | Two network sets with 1000 each x /32s | **Efficient**
- 2 iptables/eBPF rules
- 2 IP set with 1000 entries | +| source:
nets: [... 2000 /32s ...]
- source:
nets: [1 x /32]
- source: nets: [1 x /32]
- ... x 2000 | Not used | **Inefficient**
Results in programming 2k iptables/eBPF rules
- 2000+ iptables/eBPF rules
- 0 IP sets | + +For more examples of network sets with policy, see: + +- Namespaced network set + + - [Secure ingress access to a microservice or application](../applications/index.mdx) + +- Global network set + - [Secure egress access from workloads to destinations outside the cluster](../applications/egress-controls.mdx) + - [Global egress access controls](../enterprise-security/global-egress.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/calico-cloud-features/projects.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/calico-cloud-features/projects.mdx new file mode 100644 index 0000000000..4439438e36 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/calico-cloud-features/projects.mdx @@ -0,0 +1,59 @@ +--- +description: Step-by-step tutorial for creating Calico Cloud projects to group managed clusters by team, geography, or environment and scope observability to that grouping. +--- + +import Screenshot from '/src/___new___/components/Screenshot'; + + +# Projects + +You can use projects to organize your managed clusters into logical groupings. + + + +## About projects + +In $[prodname], a *project* is a logical grouping of one or more managed clusters. +You can use projects to organize your managed clusters in ways that are useful for your organization. +For example, you can use projects to segregate clusters by product teams, or by geo location. + +When you view observability tools in Calico Cloud, you see only what is available for the current project. + +All Calico Cloud organizations start with a single default project. +You can have up to three projects at a given time. + +## Create a project + +**Prerequisites** + +* You have Admin or Owner privileges. + +**Procedure** + +1. From any page in the web console, click the **Project** menu and then click **Manage Projects**. +1. On the **Projects** page, click **Add Project**. +1. Enter a name and description for the project, and then click the **Add Project** button. + + Provisioning the new project takes a few minutes. + When provisioning is complete, you can begin connecting clusters. + +## Connect a cluster to a project + +* To connect a cluster to a specific project, make sure you have selected the project you want as you follow the procedure to [connect your cluster to Calico Cloud](../../get-started/install-cluster.mdx). + +## Move a managed cluster to another project + +You can move a managed cluster to another project by removing it from the original project and connecting it to the new project. + +:::warning +Log data held by Calico Cloud will be deleted when you remove your cluster from the original project. +When you reconnect your cluster to Calico Cloud as part of the new project, flow logs will start to be collected again. +You will not be able to restore access to the old logs. + +If you need to retain logs, contact [our support team](https://tigeraio.my.site.com/community/s/login/). +::: + +1. To remove your cluster from Calico Cloud, find your cluster on the **Managed Clusters** page, and then click **Actions > Delete**. + After a moment, the cluster is removed from the list. +1. Move to the new project by clicking the **Projects** menu and selecting your project. +1. To reconnect your cluster, follow the guide to [connect your cluster to Calico Cloud](../../get-started/install-cluster.mdx). diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/calico-cloud-features/service-graph.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/calico-cloud-features/service-graph.mdx new file mode 100644 index 0000000000..7eb0258197 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/calico-cloud-features/service-graph.mdx @@ -0,0 +1,132 @@ +--- +description: Step-by-step tutorial for the Calico Cloud Service Graph that visualizes pod, service, and namespace communication across a managed cluster. +--- + +# Service Graph tutorial + +import ReactPlayer from 'react-player'; + +:::note + +This tutorial references the labs cluster setup that is part of the $[prodname] Trial environment. + +::: + +## What you will learn + +- The basics of Service Graph +- To visualize workload communication in the labs cluster using Service Graph +- Features of Service Graph that help you manage the scale of clusters with a large number of namespaces + +## The what and why of Service Graph + +One thing we consistently hear from DevOps engineers, SREs, and platform operators is that they struggle with getting basic visibility within their Kubernetes cluster. As more apps and microservices are deployed, and workloads are changing continuously, it becomes hard to understand how everything is working together in such a complex multi-tenant environment. Service Graph provides a "weather map" view of everything in a cluster, helping new team members ramp up quickly on how everything is communicating with each other, and an easy way to engage the right stakeholders when issues come up. Service Graph delivers a point-to-point, topographical view of how namespaces, services, and deployments are communicating with each other. + +## Quick demo + +**12-minute video** + + + +## Exploring Service Graph with the labs cluster + +> Select the **tigera-labs cluster** in the upper right corner of the web console. +> In the left navigation menu, select **Service Graph**, **Default**. + +The default view in Service Graph provides a representation of how namespaces within the cluster are communicating with each other. Each namespace is represented with the standard Kubernetes icon for namespaces. There are some demo namespaces - **hipstershop**, **storefront**, and **acme** - and other Tigera namespaces that are used for $[prodname]. Egress traffic leaving the cluster is represented with globe icons. Under the hood, each of these globe icons is a custom resource called a **NetworkSet**. + +### Network sets + +By default, $[prodname] provides two system-defined network sets - one for public IPs called **public network** and another for private IPs called **private network**. You can also define your own network sets with a list of IPs or CIDRs and they will appear on this graph, enabling you to get more granular visibility into the egress traffic that is specific to your environment. The **public-ip-range** is an example of such a network set that has been already created in the labs cluster. + +For more examples of using network sets, see [Secure egress access from workloads to destinations outside the cluster](../applications/egress-controls.mdx), and [Understanding network sets](networksets.mdx). + +### Selecting graph edges and graph nodes + +> Click the `<<` tab in the upper right to open the details panel. + +The details panel provides additional information on a selected graph node or edge. The log panel below the graph also automatically updates and/or filters data (if available) for the selection. + +> Click on the edge between the **hipstershop** namespace, and the network set **public network**. The flow logs are automatically filtered below in the Flows tab, and you can expand any of the rows in that tab to view some of the detailed metadata that $[prodname] provides around workload communication. + +### Double-clicking on a namespace + +You can also double-click on a namespace to see services and deployments within a namespace, and how they are communicating externally, and with the rest of the cluster. + +> Double-click **hipstershop** or **storefront**. All of the resources in purple are part of the selected namespace (which is also listed in the breadcrumb in the upper left), and anything external to the namespace is represented in blue. These views are useful when troubleshooting a specific application or microservice running in your cluster. + +> To return the default view, click the **Namespaces** breadcrumb in the upper left. + +### Right-click actions to manage scale + +Right-clicking a namespace, service, or deployment brings up another set of actions that are designed to help you manage the scale of your Kubernetes cluster. Although the labs cluster has only a dozen or so namespaces, your Kubernetes clusters could likely have over one hundred namespaces. + +> Right-click **hipstershop**, and select **Hide unrelated**. + +Hide unrelated allows you to quickly filter and trim the Service Graph to show just the selected entity and anything it is communicating with. This is helpful in troubleshooting issues for a specific namespace or application, and for application teams to quickly understand their upstream and downstream dependencies. + +> To reset the view after this action, select the **Reset icon** (vertical column of icons), and click, **Reset view & filters**. + +## Using layers and views + +Another unique feature of Service Graph is the concept of **layers**, which allows you to create meaningful groupings of resources so you can easily hide and show them on the graph. It is another tool to help you manage the overall scale of visualizing workloads within your cluster. You can create layers for different types of platform infrastructure you might have in your cluster - networking, storage, logging. + +> Click the `>>` tab in the upper left (next to Namespaces) to open the panel. Expand the Tigera components layer to view its namespaces. + +The layer called **Tigera components** contains namespaces related to $[prodname]. Click the ellipsis/spillover menu on the right, and select **Hide layer**. + +This hides the entire layer on the graph, making it easy to hide/show a group of related namespaces. + +> Reset the view by selecting, **Restore layer**. + +### Views + +**Views** allow you to save the state of the graph on the canvas. + +> Click the **Views** (panel above Layers). + +Let's create a view that shows only items related to the hipstershop product catalog. + +> Double-click **hipstershop**, select the _productcatalogservice_, right-click and select **Hide unrelated to service group**. In the View tab, click **Save Current** and save the view as "Product Catalog”. + +> Click the **Namespaces** breadcrumb to reset the views and filters. In the Views tab, you can see your saved view, "Product Catalog" -- so you can quickly revisit this view at any time. + +## Logs and details panel + +As you are working with Service Graph, the bottom panel and the details panel (right of the main canvas) will display additional information for selections you make on the graph. The exact information displayed will vary depending on the following: + +- Whether you select a graph node or edge +- Type of graph node selected (namespace, network set, service, etc.) +- Whether or not you have deployed Envoy for Layer 7 visibility + +### Details panel + +> Double-click the **hipstershop** namespace to see a detailed view of service-to-service communication. Select the **checkoutservice** on the graph. + +You can see the **Inbound** and **Outbound** connections for this service. Expanding any of those connections (hovering over the line and clicking the arrow) shows you volumetric data for each of those flows. In the **Insights** section, you can click on items like DNS stats in use for the service. + +> In the line, `checkoutservice >> shippingservice`, click the green arrows to display protocols and policies. + +The **Process Info** section shows you specific processes associated with the traffic flows for the `checkoutservice`. This can be helpful in troubleshooting scenarios where identifying the specific binaries that are involved in service communication may be required to track down a bug. + +### Logs panel + +In the bottom panel are several tabs: Flows, DNS, HTTP, Alerts, and Capture Jobs. + +With the `checkoutservice` still selected, filters are automatically applied on the Flows tab with detailed logs for Layer 4 traffic. Similar filters are automatically applied to the DNS and HTTP flows. Note that HTTP flows require the deployment of Envoy to see Layer 7 traffic in any cluster connected to $[prodname]. + +The Alerts tab will show alerts if they are generated, and Capture Jobs will show any packet capture jobs that have been defined for this namespace. + +Now that you understand the basics for Service Graph, we recommend: + +- [Understanding policy tiers](../../network-policy/policy-tiers/tiered-policy.mdx) +- [Understanding network sets](networksets.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/calico-cloud-features/tour.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/calico-cloud-features/tour.mdx new file mode 100644 index 0000000000..485894b7c4 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/calico-cloud-features/tour.mdx @@ -0,0 +1,204 @@ +--- +description: Step-by-step tutorial for the Calico Cloud web console interface, walking the left navbar from dashboards to policy, observability, threat defense, and image scanning. +--- + +# Web console tutorial + +## What you will learn + +- Web console features and controls +- How to gain visibility into clusters + +Let's go through each item in the web console left navbar from top to bottom. You can follow along using any cluster. + +## Dashboards + +> From the left navbar, click Dashboards. + +Dashboards help you see and understand the activity in your cluster. +Each dashboard is made up of categorized graphs, charts, and diagrams that visually represent the data in your logs. + +![dashboards](/img/calico-enterprise/dashboards.png) + + +* The **Cluster Health** dashboard provides a birds-eye view of cluster activity. + +* The **Traffic Volume** dashboard provides a high-level view of traffic in your cluster. +* The **DNS Logs** dashboard summarizes DNS data and logs into metrics, providing high-level information on the types of DNS lookups made, responses, and overall DNS performance. +* The **Flow Logs** dashboard gives you an overview of how packets are being sent and received by all the pods in your cluster. +* The **HTTP Traffic** dashboard provides application performance metrics for in-scope Kubernetes services. +* The **Network Performance** dashboard provides TCP metrics to help you identify bottlenecks, packet loss, and performance issues. + +## Service Graph + +> From the left navbar, select **Service Graph**, **Default** + +Service Graph provides a point-to-point, topographical representation of network traffic within your cluster. It is the primary tool for visibility and troubleshooting. + +![service-graph](/img/calico-cloud/service-graph.png) + +**Namespaces** + +Namespaces are the default view in Service Graph. + +When you expand the top right panel `<<`, you see a detailed view of the service-to-service communications for the namespace. + +![service-graph-namespace](/img/calico-cloud/service-graph-namespace.png) + +**Nodes and edges** + +Lines going to/from nodes are called edges. When you click on a node or edge, the right panel shows details, and the associated flow logs are automatically filtered in the bottom panel. + +![edges](/img/calico-cloud/edges.png) + +**Layers** + +Layers allow you to create meaningful groupings of resources so you can easily hide and show them on the graph. For example, you can group resources for different platform infrastructure types in your cluster like networking, storage, and logging. + +> Click the panel on the left (`>>`) by the Namespaces breadcrumb, and then expand the Tigera components layer. + +![service-graph-layers](/img/calico-cloud/service-graph-layers.png) + +The **Tigera components** layer contains namespaces for $[prodname] networking components, and a view of interest to Dev/Ops. + +> Click the vertical ellipses and select, **Hide layer**. Notice that only the business application namespaces remain visible in the graph. + +> To make this layer less visible, select **Restore layer**and click **De-emphasize layer**. + +**Logs, alerts, and capture jobs** + +The panel at the bottom below the graph provides tools for troubleshooting connectivity and performance issues. **Logs** (Flows, DNS, and HTTP) are the foundation of security and observability in $[prodname]. When you select a node or edge in the graph, logs are filtered for the node or service. For example, here is a flow log with details including how the policies were processed in tiers. + +![service-graph-flows](/img/calico-cloud/service-graph-flows.png) + +**Alerts** + +For convenience, the Alerts tab duplicates the alerts you have enabled in the **Alerts tab** in the left navbar. By default, alerts are not enabled. + +**Capture jobs** + +Service Graph integrates a packet feature for capturing traffic for a specific namespace, service, replica set, daemonset, statefulset, or pod. You can then download capture files to your favorite visualization tool like Wireshark. + +> Right-click on any endpoint to start or schedule a capture. + +![packet-capture-service](/img/calico-cloud/packet-capture-service.png) + +**Flow Visualizations** + +> From the left navbar, select **Service Graph**, **Flow Visualizations**. + +Flow Visualizer (also called, "FlowViz") is a $[prodname] tool for drilling down into network traffic within the cluster to troubleshoot issues. The most common use of Flow Visualizer is to drill down and pinpoint which policies are allowing and denying traffic between services. + +![flow-viz](/img/calico-cloud/flow-viz.png) + +## Policies + +> From the left navbar, click **Policies**. + +Network policy is the primary tool for securing a Kubernetes network. Policy is used to restrict network traffic (egress and ingress) in your cluster so only the traffic that you want to flow is allowed. $[prodname] supports these policies: + +- $[prodname] network policy +- $[prodname] global network policy +- Kubernetes policy + +$[prodname] uses **tiers** (also called, hierarchical tiers) to provide guardrails for managing network policy across teams. Policy tiers allow users with more authority (for example, Dev/ops user) to enforce network policies that take precedence over teams (for example, service owners and developers). + +**Policies Board** is the default view for managing tiered policies. + +![policy-board](/img/calico-cloud/policy-export.png) + +Users typically use a mix of Policy Board and YAML files. Note that you can export one or all policies in a tier to YAML. + +The **Policy Board filter** lets you filter by name, namespace, policy types and label selectors. + +![policy-filters](/img/calico-cloud/policy-search.png) + +The following features provide more security and guardrails for teams. + +**Recommended a policy** + +> In Policies Board, click **Recommend a policy**. + +One of the first things you'll want to do after installation is to secure unprotected pods/workloads with network policy. (For example, Kubernetes pods allow traffic from any source by default.) The Recommend a policy feature generates policies that protect specific endpoints in the cluster. Users with minimal experience with network policy can easily get started. + +![recommend-policy](/img/calico-cloud/recommend-policy.png) + +**Policy stage** + +When you create a policy, it is a best practice to stage it to evaluate the effects before enforcing it. After you verify that a staged network policy is allowing traffic as expected, you can enforce it. + +**Preview** + +When you edit a policy, you can select **Preview** to see how changes may affect existing traffic. + +![policy-preview](/img/calico-cloud/policy-preview-3.24.png) + +## Endpoints + +> From the left navbar, click **Endpoints**. + +**Endpoint Details** + +This page is a list of all pods in the cluster (also known as workload endpoints). + +![endpoints](/img/calico-cloud/endpoint.png) + +**Node List** + +This page lists all nodes associated with your cluster. + +![node-list](/img/calico-cloud/node-list.png) + +## Network Sets + +Network sets and global network sets are $[prodname] resources for defining IP subnetworks/CIDRs, which can be matched by standard label selectors in policy (Kubernetes or $[prodname]). They are a powerful feature for use/reuse and scaling policy. + +A simple use case is to limit traffic to/from external networks. For example, you can create a global network set with "deny-list CIDR ranges 192.0.2.55/32 and 203.0.113.0/24", and then reference the network set in a global network policy. This also allows you to see this traffic in Service Graph. + +![networksets](/img/calico-cloud/networksets.png) + +## Managed clusters + +> From the left navbar, click **Managed clusters**. + +This page is where you switch views between clusters in the web console. When you connect to a different cluster, the entire Manager view changes to reflect the selected cluster. + +![managed-clusters](/img/calico-cloud/managed-clusters.png) + +## Activity + +> From the left navbar, select **Activity**, **Timeline**. + +**Timeline** + +What changed, who did it, and when? This information is critical for security. Native Kubernetes doesn’t provide an easy way to capture audit logs for pods, namespaces, service accounts, network policies, and endpoints. The $[prodname] timeline provides audit logs for all changes to network policy and other resources associated with your $[prodname] deployment. + +![timeline](/img/calico-cloud/timeline.png) + +> From the left navbar, selection **Activity**, **Alerts**. + +**Alerts** + +How do you know if you have an infected workload? A possible threat? $[prodname] detects and alerts on unexpected network behavior that may indicate a security breach. You can create alerts for: + +- Known attacks and exploits (for example, exploits found at Shopify, Tesla, Atlassian) +- DOS attempts +- Attempted connections to botnets and command and control servers + +![alerts](/img/calico-cloud/alerts.png) + +## Logs + +$[prodname] includes a fully-integrated deployment of Elastic to collect flow log data that drives key features like the Flow Visualizer, metrics in the Dashboard and Policy Board, policy automation, and testing features and security. + +> From the left navbar, click **Logs**. + +## Threat feeds + +You can add threat intelligence feeds to $[prodname] to trace network flows of suspicious IP addresses and domains. Then, you can use network policy to block pods from contacting IPs or domains. + +Now that you understand the basics, we recommend the following: + +- [Service Graph tutorial](service-graph.mdx) +- [Understanding policy tiers](../../network-policy/policy-tiers/tiered-policy.mdx) +- [Understanding network sets](networksets.mdx) diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/enterprise-security/default-deny.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/enterprise-security/default-deny.mdx new file mode 100644 index 0000000000..2238fd2a4c --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/enterprise-security/default-deny.mdx @@ -0,0 +1,78 @@ +--- +description: Step-by-step tutorial for staging and rolling out a global default-deny policy in Calico Cloud so unwanted ingress and egress is blocked across the cluster. +--- + +# Global default deny policy + +In this article you will learn when and how to create a global default deny policy for the cluster. + +## What is it, when should you create one? + +A global default deny policy ensures that unwanted traffic (ingress and egress) is denied by default. Pods without policy (or incorrect policy) are not allowed traffic until appropriate network policy is defined. Although the staging policy tool will help you find incorrect and missing policy, a global deny helps mitigate against other lateral malicious attacks. + +## Best practice #1: Allow, stage, then deny + +We recommend that you create a global default deny policy _after you complete writing policy for the traffic that you want to allow_. Use the stage policy feature to get your allowed traffic working as expected, then lock down the cluster to block unwanted traffic. The following steps summarizes the best practice: + +1. Create a staged global default deny policy. It will shows all the traffic that would be blocked if it were converted into a deny. +1. Create other network policies to individually allow the traffic shown as blocked in step 1, until no connections are denied. +1. Convert the staged global network policy to an enforced policy. + +## Best practice #2: Keep the scope to non-system pods + +A global default deny policy applies to the entire cluster including all workloads in all namespaces, hosts (computers that run the hypervisor for VMs, or container runtime for containers), including Kubernetes control plane and $[prodname] control plane nodes and pods. + +For this reason, the best practice is to create a global default deny policy for **non-system pods** as shown in the following example. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: deny-app-policy +spec: + namespaceSelector: has(projectcalico.org/name) && projectcalico.org/name not in {"kube-system", "calico-system"} + types: + - Ingress + - Egress + egress: + # allow all namespaces to communicate to DNS pods + - action: Allow + protocol: UDP + destination: + selector: 'k8s-app == "kube-dns"' + ports: + - 53 + - action: Allow + protocol: TCP + destination: + selector: 'k8s-app == "kube-dns"' + ports: + - 53 +``` + +Note the following: + +- Even though we call this policy "global default deny", the above policy is not explicitly _denying traffic_. By selecting the traffic with the `namespaceSelector` but not specifying an allow, the traffic is denied after all other policy is evaluated. This design also makes it unnecessary to ensure any specific order (priority) for the default-deny policy. +- Allowing access to `kube-dns` simplifies per-pod policies because you don't need to duplicate the DNS rules in every policy +- The policy deliberately excludes the `kube-system` and `calico-system` namespaces by using a negative `namespaceSelector` to avoid impacting any control plane components + +Next, add the policy to the default tier. (As noted above, anywhere in the default tier is fine.) + +Next, use the stage policy feature and verify that the policy does not block any necessary traffic before enforcing it. + +### Don't try this! + +The following policy looks fine on the surface, and it does works. But as described in Best practices #2, the policy could break your cluster because the scope is too broad. Therefore, we do not recommend adding this type of policy to the default tier, even if you have verified allowed traffic using the stage policy feature. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: default.default-deny +spec: + tier: default + selector: all() + types: + - Ingress + - Egress +``` diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/enterprise-security/global-egress.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/enterprise-security/global-egress.mdx new file mode 100644 index 0000000000..9d183c41ba --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/enterprise-security/global-egress.mdx @@ -0,0 +1,90 @@ +--- +description: Step-by-step tutorial for building cluster-wide egress access controls in Calico Cloud using global network sets, domains, and team-scoped policy. +--- + +# Global egress access controls + +In this article you will learn how to implement egress access controls cluster-wide for all applications and microservices. + +In this example, we will implement global egress access controls for **dev1 team**: + +- Egress access control for all applications managed by dev1: applications (**app1**) and microservices (**app2**) +- dev1 pods can egress access to a repo named, `repo.acme.corp` at port 443 +- dev1 pods can egress access to a trusted partner (in this example, another business unit in the organization) at `10.10.10.10` on port 1010. + +![global-egress](/img/calico-cloud/global-egress.png) + +## Create global network sets + +First, we need to create a GlobalNetworkSet for the trusted repo, `repo.acme.corp`. Because we already have a label taxonomy following the best practices, it is easy. Just use the `allowedEgressDomains` to specify the trusted repo, `repo.acme.corp`. + +```yaml +kind: GlobalNetworkSet +apiVersion: projectcalico.org/v3 +metadata: + name: trusted-repo + labels: + trusted-ep: dev1-repo +spec: + allowedEgressDomains: + - repo.acme.corp +``` + +Next, we create a separate global network set for our trusted business unit within the organization. We use the `dev1-partners` label, and specify the IP address, `10.10.10.10/32`. + +```yaml +kind: GlobalNetworkSet +apiVersion: projectcalico.org/v3 +metadata: + name: trusted-partners + labels: + trusted-ep: dev1-partners +spec: + nets: + - 10.10.10.10/32 +``` + +## Create global network policy + +Before we create our GlobalNetworkPolicy, let's review our labels and set up. + +- We assume that app1 and app2 and all other dev1 team apps’ pods have a label that identifies dev1 (tenant: dev1) to enforce the declared controls. + +- For selected pods (all dev1 pods), all egress traffic that is not explicitly allowed is denied, as part of the best practices for policy tiers. + +- The policy below, which is allowing egress communication of dev1 pods with all other pods in the cluster, is done by selecting all namespaces as the destination. This means we are applying granular controls to pods for traffic destined to external endpoints, and we are allowing egress traffic destined to other pods in the cluster. This is recommended unless you have specific control requirements that dictate otherwise. This simplifies policy creation since we have already defined granular ingress controls for intra-cluster pod-to-pod communication. The alternative would be to define granular ingress and egress controls to all pods, which adds complexity in policy development. + +Here is our GlobalNetworkPolicy. The dev1 team is allowed egress access to the trusted repo, `dev-repo` via port 443, and also to the trusted business unit, `dev-partners` via port 1010. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: security.egress-dev1 +spec: + tier: security + selector: tenant == "dev1" + egress: + - action: Allow + protocol: TCP + source: {} + destination: + selector: trusted-ep == "dev1-repo" + ports: + - '443' + - action: Allow + protocol: TCP + source: {} + destination: + selector: trusted-ep == "dev1-partners" + ports: + - '1010' + - action: Allow + source: {} + destination: + namespaceSelector: all() + types: + - Egress +``` + +As you can see, once you have your labels in place, creating policy to secure teams is straightforward. diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/enterprise-security/index.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/enterprise-security/index.mdx new file mode 100644 index 0000000000..db3521f181 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/enterprise-security/index.mdx @@ -0,0 +1,11 @@ +--- +description: Index of Calico Cloud tutorials that build enterprise security controls including namespace isolation, global egress, default deny, and platform-tier policy. +hide_table_of_contents: true +--- + +# Implement enterprise security controls + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/enterprise-security/namespace-isolation.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/enterprise-security/namespace-isolation.mdx new file mode 100644 index 0000000000..7f380e3f06 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/enterprise-security/namespace-isolation.mdx @@ -0,0 +1,88 @@ +--- +description: Step-by-step tutorial for isolating namespaces in Calico Cloud using global network policies with Pass rules in a security tier across business units and environments. +--- + +# Namespace isolation and access controls + +In this article you will learn how to create $[prodname] global network policies to isolate namespaces for ingress traffic. + +## Isolate access + +Isolating ingress traffic access can be implemented across different groups in your organization, for example: + +- Business unit, development team, or project +- Deployment environment (Dev/Stage/Prod or Pre-prod/Prod) +- Control plane +- Management plane +- Compliance environment + +## Create global network policies + +Microsegmentation across namespaces is achieved using $[prodname] policy labels and selectors with a **Pass** action rule. If you are unfamiliar with policy action rules, see [Tutorial - Understanding policy tiers](../../network-policy/policy-tiers/tiered-policy.mdx). + +In the following example, let's assume the following: + +- We created a **security tier** (recommended for your enterprise security controls like multi-tenancy) +- Two development teams (`dev1` and `dev2`), each managing two applications or microservices +- The security tier will delegate (pass) microsegmentation controls to the application tier +- We will use a GlobalNetworkPolicy with cluster wide scope that applies to pods in all namespaces in the cluster. + +For example: + +![multi-tenancy-ingress](/img/calico-cloud/multi-tenancy-ingress.png) + +**GlobalNetworkPolicy 1** + +The following policy is for ingress traffic that goes through a trusted load balancer. It resides in the security tier, and uses the **Pass** action rule to delegate control to the application tier for the `dev1` tenant. Because the application tier contains endpoints that apply, traffic is processed. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: security.ingress-dev1 +spec: + tier: security + order: 200 + selector: tenant == "dev1" + ingress: + - action: Pass + source: + selector: trusted-ep == "load-balancer" + - action: Pass + source: + selector: tenant == "dev1" + types: + - Ingress +``` + +**GlobalNetworkPolicy 2** + +Similarly, we create a second GlobalNetworkPolicy for `dev2` tenant that delegates control to the application tier for the `dev2` applications. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: security.ingress-dev2 +spec: + tier: security + order: 300 + selector: tenant == "dev2" + ingress: + - action: Pass + source: + selector: trusted-ep == "load-balancer" + - action: Pass + source: + selector: tenant == "dev2" + types: + - Ingress +``` + +## Summary and recommendations + +In these examples, we implemented ingress access controls for two development teams. The global network policy has cluster-wide scope (applying to pods in all namespaces in the cluster). For **egress controls**, we recommend the following: + +- Use permissive egress policies for intra-cluster communication +- Implement [Global default deny](default-deny.mdx) security controls +- Implement [Global egress access controls](global-egress.mdx) for communication with endpoints external to the cluster in the security tier. These controls govern the communication of all applications for a given tenant with the outside world. diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/enterprise-security/platform.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/enterprise-security/platform.mdx new file mode 100644 index 0000000000..a26d6bd3d1 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/enterprise-security/platform.mdx @@ -0,0 +1,60 @@ +--- +description: Step-by-step tutorial for securing platform-tier applications in Calico Cloud with ingress and egress controls for storage, secrets, and monitoring workloads. +--- + +# Platform application access controls + +In this article you will learn some best practices to secure platform tier applications. + +## Isolate platform applications + +You may have several types of platform applications in your environment: + +- Storage platform +- Secret management +- Container security +- Platform monitoring +- Application performance monitoring + +We recommend that you implement all your platform applications controls in a **platform tier**, implicitly allowing granular ingress and egress controls for platform applications pods and denying everything else. This effectively isolates platform applications from business applications, and extends the multi-tenancy and application microsegmentation controls to platform applications. + +## Platform order + +```yaml +apiVersion: projectcalico.org/v3 +kind: Tier +metadata: + name: platform +spec: + order: 200 +``` + +Order 200 implies that the platform tier comes before the security tier and that its policies are processed first. Some organizations require that security team controls take precedence over other controls. In that case, the security tier policies need to pass controls to the platform tier for granular control. + +## Map your application access controls to policy + +Next, we recommend that you refer to your platform application documentation and get the communication map of application pods, and reflect the controls in your network policy. + +## Create a global network policy + +Here is our global network policy that allows application (`platform-app1`) and service (`platf-app1-svc2`), ingress access to service (`svc = platf-app1-svc1`) from port 12345. + +```yaml +apiVersion: projectcalico.org/v3 +kind: GlobalNetworkPolicy +metadata: + name: platform.platform-app2-svc2 +spec: + tier: platform + selector: (app == "platform-app1" && svc == "platf-app1-svc2") + ingress: + - action: Allow + protocol: TCP + source: + selector: svc == "platf-app1-svc1" + destination: + ports: + - '12345' + types: + - Ingress +``` diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/index.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/index.mdx new file mode 100644 index 0000000000..152da003af --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/index.mdx @@ -0,0 +1,53 @@ +--- +description: Index of step-by-step tutorials for Calico Cloud features, ingress and egress policy, enterprise tier controls, and Kubernetes networking fundamentals. +title: Tutorials +hide_table_of_contents: true +--- +import { DocCardLink, DocCardLinkLayout } from '/src/___new___/components'; + +# Tutorials + +Learn more about Calico Cloud and Kubernetes network policy. + +## Calico Cloud features + + + + + + + +## Secure ingress and egress for applications + + + + + + +## Implement enterprise security controls + + + + + + + + +## Kubernetes networking for beginners + + + + + + + + +## Kubernetes tutorials and demos + + + + + + + + diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/kubernetes-tutorials/index.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/kubernetes-tutorials/index.mdx new file mode 100644 index 0000000000..e3bb15379f --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/kubernetes-tutorials/index.mdx @@ -0,0 +1,11 @@ +--- +description: Index of Calico Cloud tutorials and demos covering Kubernetes network policy syntax, basic and advanced policy patterns, and the stars policy visualizer. +hide_table_of_contents: true +--- + +# Kubernetes tutorials and demos + +import DocCardList from '@theme/DocCardList'; +import { useCurrentSidebarCategory } from '@docusaurus/theme-common'; + + diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/kubernetes-tutorials/kubernetes-demo.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/kubernetes-tutorials/kubernetes-demo.mdx new file mode 100644 index 0000000000..edc7406a38 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/kubernetes-tutorials/kubernetes-demo.mdx @@ -0,0 +1,103 @@ +--- +description: Step-by-step tutorial for running the stars demo in Calico Cloud to visualize how Kubernetes network policy allows and denies frontend, backend, and client traffic. +--- + +# Kubernetes policy, demo + +The included demo sets up a frontend and backend service, as well as a client service, all +running on Kubernetes. It then configures network policy on each service. + +{/*TODO-XREFS-CC +## Prerequisites + +To create a Kubernetes cluster which supports the Kubernetes network policy API, follow +one of our [getting started guides](/getting-started/index). +*/} + +## Running the stars example + +### 1) Create the frontend, backend, client, and management-ui apps. + +```shell +kubectl create -f $[tutorialFilesURL]/00-namespace.yaml +kubectl create -f $[tutorialFilesURL]/01-management-ui.yaml +kubectl create -f $[tutorialFilesURL]/02-backend.yaml +kubectl create -f $[tutorialFilesURL]/03-frontend.yaml +kubectl create -f $[tutorialFilesURL]/04-client.yaml +``` + +Wait for all the pods to enter `Running` state. + +```bash +kubectl get pods --all-namespaces --watch +``` + +> Note that it may take several minutes to download the necessary Docker images for this demo. + +The management UI runs as a `NodePort` Service on Kubernetes, and shows the connectivity +of the Services in this example. + +You can view the UI by visiting `http://:30002` in a browser. + +Once all the pods are started, they should have full connectivity. You can see this by visiting the UI. Each service is +represented by a single node in the graph. + +- `backend` -> Node "B" +- `frontend` -> Node "F" +- `client` -> Node "C" + +### 2) Enable isolation + +Running following commands will prevent all access to the frontend, backend, and client Services. + +```shell +kubectl create -n stars -f $[tutorialFilesURL]/default-deny.yaml +kubectl create -n client -f $[tutorialFilesURL]/default-deny.yaml +``` + +#### Confirm isolation + +Refresh the management UI (it may take up to 10 seconds for changes to be reflected in the UI). +Now that we've enabled isolation, the UI can no longer access the pods, and so they will no longer show up in the UI. + +### 3) Allow the UI to access the services using network policy objects + +Apply the following YAML files to allow access from the management UI. + +```shell +kubectl create -f $[tutorialFilesURL]/allow-ui.yaml +kubectl create -f $[tutorialFilesURL]/allow-ui-client.yaml +``` + +After a few seconds, refresh the UI - it should now show the Services, but they should not be able to access each other anymore. + +### 4) Create the backend-policy.yaml file to allow traffic from the frontend to the backend + +```shell +kubectl create -f $[tutorialFilesURL]/backend-policy.yaml +``` + +Refresh the UI. You should see the following: + +- The frontend can now access the backend (on TCP port 6379 only). +- The backend cannot access the frontend at all. +- The client cannot access the frontend, nor can it access the backend. + +### 5) Expose the frontend service to the client namespace + +```shell +kubectl create -f $[tutorialFilesURL]/frontend-policy.yaml +``` + +The client can now access the frontend, but not the backend. Neither the frontend nor the backend +can initiate connections to the client. The frontend can still access the backend. + +To use $[prodname] to enforce egress policy on Kubernetes pods, see [the advanced policy demo](kubernetes-policy-advanced.mdx). + +### 6) (Optional) Clean up the demo environment + +You can clean up the demo by deleting the demo Namespaces: + +```bash +kubectl delete ns client stars management-ui +``` diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/kubernetes-tutorials/kubernetes-network-policy.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/kubernetes-tutorials/kubernetes-network-policy.mdx new file mode 100644 index 0000000000..39797dfe67 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/kubernetes-tutorials/kubernetes-network-policy.mdx @@ -0,0 +1,179 @@ +--- +description: Step-by-step tutorial for learning Kubernetes network policy concepts with Calico Cloud, covering ingress, egress, selectors, and policy enforcement. +--- + +# Get started with Kubernetes network policy + +## Big picture + +Kubernetes network policy lets administrators and developers enforce which network traffic is allowed using rules. + +## Value + +Kubernetes network policy lets developers secure access to and from their applications using the same simple language they use to deploy them. Developers can focus on their applications without understanding low-level networking concepts. Enabling developers to easily secure their applications using network policies supports a shift left DevOps environment. + +## Concepts + +The Kubernetes Network Policy API provides a standard way for users to define network policy for controlling network traffic. However, Kubernetes has no built-in capability to enforce the network policy. To enforce network policy, you must use a network plugin such as Calico. + +### Ingress and egress + +The bulk of securing network traffic typically revolves around defining egress and ingress rules. From the point of view of a Kubernetes pod, **ingress** is incoming traffic to the pod, and **egress** is outgoing traffic from the pod. In Kubernetes network policy, you create ingress and egress “allow” rules independently (egress, ingress, or both). + +### Default deny/allow behavior + +**Default allow** means all traffic is allowed by default, unless otherwise specified. +**Default deny** means all traffic is denied by default, unless explicitly allowed. + +## How to + +Before you create your first Kubernetes network policy, you need to understand the default network policy behaviors. If no Kubernetes network policies apply to a pod, then all traffic to/from the pod are allowed (default-allow). As a result, if you do not create any network policies, then all pods are allowed to communicate freely with all other pods. If one or more Kubernetes network policies apply to a pod, then only the traffic specifically defined in that network policy are allowed (default-deny). + +You are now ready to start fine-tuning traffic that should be allowed. + +- [Create ingress policies](#create-ingress-policies) +- [Allow ingress traffic from pods in the same namespace](#allow-ingress-traffic-from-pods-in-the-same-namespace) +- [Allow ingress traffic from pods in a different namespace](#allow-ingress-traffic-from-pods-in-a-different-namespace) +- [Create egress policies](#create-egress-policies) +- [Allow egress traffic from pods in the same namespace](#allow-egress-traffic-from-pods-in-the-same-namespace) +- [Allow egress traffic to IP addresses or CIDR range](#allow-egress-traffic-to-ip-addresses-or-cidr-range) +- [Best practice: create deny-all default network policy](#best-practice-create-deny-all-default-network-policy) +- [Create deny-all default ingress and egress network policy](#create-deny-all-default-ingress-and-egress-network-policy) + +### Create ingress policies + +Create ingress network policies to allow inbound traffic from other pods. + +Network policies apply to pods within a specific **namespace**. Policies can include one or more ingress rules. To specify which pods in the namespace the network policy applies to, use a **pod selector**. Within the ingress rule, use another pod selector to define which pods allow incoming traffic, and the **ports** field to define on which ports traffic is allowed. + +#### Allow ingress traffic from pods in the same namespace + +In the following example, incoming traffic to pods with label **color=blue** are allowed only if they come from a pod with **color=red**, on port **80**. + +```yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-same-namespace + namespace: default +spec: + podSelector: + matchLabels: + color: blue + ingress: + - from: + - podSelector: + matchLabels: + color: red + ports: + - port: 80 +``` + +#### Allow ingress traffic from pods in a different namespace + +To allow traffic from pods in a different namespace, use a namespace selector in the ingress policy rule. In the following policy, the namespace selector matches one or more Kubernetes namespaces and is combined with the pod selector that selects pods within those namespaces. + +:::note + +Namespace selectors can be used only in policy rules. The **spec.podSelector** applies to pods only in the same namespace as the policy. + +::: + +In the following example, incoming traffic is allowed only if they come from a pod with label **color=red**, in a namespace with label **shape=square**, on port **80**. + +```yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-different-namespace + namespace: default +spec: + podSelector: + matchLabels: + color: blue + ingress: + - from: + - podSelector: + matchLabels: + color: red + namespaceSelector: + matchLabels: + shape: square + ports: + - port: 80 +``` + +### Create egress policies + +Create egress network policies to allow outbound traffic from pods. + +#### Allow egress traffic from pods in the same namespace + +The following policy allows pod outbound traffic to other pods in the same namespace that match the pod selector. In the following example, outbound traffic is allowed only if they go to a pod with label **color=red**, on port **80**. + +```yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-egress-same-namespace + namespace: default +spec: + podSelector: + matchLabels: + color: blue + egress: + - to: + - podSelector: + matchLabels: + color: red + ports: + - port: 80 +``` + +#### Allow egress traffic to IP addresses or CIDR range + +Egress policies can also be used to allow traffic to specific IP addresses and CIDR ranges. Typically, IP addresses/ranges are used to handle traffic that is external to the cluster for static resources or subnets. + +The following policy allows egress traffic to pods in CIDR, **172.18.0.0/24**. + +```yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: allow-egress-external + namespace: default +spec: + podSelector: + matchLabels: + color: red + egress: + - to: + - ipBlock: + cidr: 172.18.0.0/24 +``` + +### Best practice: create deny-all default network policy + +To ensure that all pods in the namespace are secure, a best practice is to create a default network policy. This avoids accidentally exposing an app or version that doesn’t have policy defined. + +#### Create deny-all default ingress and egress network policy + +The following network policy implements a default **deny-all** ingress and egress policy, which prevents all traffic to/from pods in the **policy-demo** namespace. Note that the policy applies to all pods in the policy-demo namespace, but does not explicitly allow any traffic. All pods are selected, but because the default changes when pods are selected by a network policy, the result is: **deny all ingress and egress traffic**. (Unless the traffic is allowed by another network policy). + +```yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: default-deny + namespace: policy-demo +spec: + podSelector: + matchLabels: {} + policyTypes: + - Ingress + - Egress +``` + +## Additional resources + +- [Kubernetes Network Policy API documentation](https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#networkpolicyspec-v1-networking-k8s-io) diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/kubernetes-tutorials/kubernetes-policy-advanced.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/kubernetes-tutorials/kubernetes-policy-advanced.mdx new file mode 100644 index 0000000000..575e319b55 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/kubernetes-tutorials/kubernetes-policy-advanced.mdx @@ -0,0 +1,342 @@ +--- +description: Step-by-step tutorial for writing advanced Kubernetes network policies in Calico Cloud, including namespace-scoped rules and default deny for ingress and egress. +--- + +# Kubernetes policy, advanced tutorial + +The Kubernetes `NetworkPolicy` API allows users to express ingress and egress policies (starting with Kubernetes 1.8.0) to Kubernetes pods +based on labels and ports. + +This guide describes how to use Kubernetes `NetworkPolicy` to define more complex network policies. + +## Requirements + +- A working Kubernetes cluster and access to it using kubectl +- Your Kubernetes nodes have connectivity to the public internet +- You are familiar with [Kubernetes NetworkPolicy](kubernetes-policy-basic.mdx) + +## Tutorial flow + +1. Create the Namespace and Nginx Service +1. Deny all ingress traffic +1. Allow ingress traffic to Nginx +1. Deny all egress traffic +1. Allow egress traffic to kube-dns +1. Cleanup Namespace + +## 1. Create the namespace and nginx service + +We'll use a new namespace for this guide. Run the following commands to create it and a plain nginx service listening on port 80. + +```bash +kubectl create ns advanced-policy-demo +kubectl create deployment --namespace=advanced-policy-demo nginx --image=nginx +kubectl expose --namespace=advanced-policy-demo deployment nginx --port=80 +``` + +### Verify access - allowed all ingress and egress + +Open up a second shell session which has `kubectl` connectivity to the Kubernetes cluster and create a busybox pod to test policy access. This pod will be used throughout this tutorial to test policy access. + +```bash +kubectl run --namespace=advanced-policy-demo access --rm -ti --image busybox /bin/sh +``` + +This should open up a shell session inside the `access` pod, as shown below. + +``` +Waiting for pod advanced-policy-demo/access-472357175-y0m47 to be running, status is Pending, pod ready: false + +If you don't see a command prompt, try pressing enter. +/ # +``` + +Now from within the busybox "access" pod execute the following command to test access to the nginx service. + +```bash +wget -q --timeout=5 nginx -O - +``` + +It should return the HTML of the nginx welcome page. + +Still within the busybox "access" pod, issue the following command to test access to google.com. + +```bash +wget -q --timeout=5 google.com -O - +``` + +It should return the HTML of the google.com home page. + +## 2. Deny all ingress traffic + +Enable ingress isolation on the namespace by deploying a [default deny all ingress traffic policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/#default-deny-all-ingress-traffic). + +```bash +kubectl create -f - < + + +Welcome to nginx!... +``` + +After creating the policy, we can now access the nginx Service. + +## 4. Deny all egress traffic + +Enable egress isolation on the namespace by deploying a [default deny all egress traffic policy](https://kubernetes.io/docs/concepts/services-networking/network-policies/#default-deny-all-egress-traffic). + +```bash +kubectl create -f - < + + +Welcome to nginx!... +``` + +Next, try to retrieve the home page of google.com. + +```bash +wget -q --timeout=5 google.com -O - +``` + +It should return: + +``` +wget: download timed out +``` + +Access to `google.com` times out because it can resolve DNS but has no egress access to anything other than pods with labels matching `app: nginx` in the `advanced-policy-demo` namespace. + +# 7. Clean up namespace + +You can clean up after this tutorial by deleting the advanced policy demo namespace. + +```bash +kubectl delete ns advanced-policy-demo +``` diff --git a/calico-cloud_versioned_docs/version-24-1/tutorials/kubernetes-tutorials/kubernetes-policy-basic.mdx b/calico-cloud_versioned_docs/version-24-1/tutorials/kubernetes-tutorials/kubernetes-policy-basic.mdx new file mode 100644 index 0000000000..053d541d75 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/tutorials/kubernetes-tutorials/kubernetes-policy-basic.mdx @@ -0,0 +1,209 @@ +--- +description: Step-by-step tutorial for writing a basic Kubernetes network policy in Calico Cloud to restrict pod-to-pod traffic using namespaces, labels, and selectors. +--- + +# Kubernetes policy, basic tutorial + +This guide provides a simple way to try out Kubernetes `NetworkPolicy` with $[prodname]. It requires a Kubernetes cluster configured with $[prodname] networking, and expects that you have `kubectl` configured to interact with the cluster. + +{/*TODO-XREFS-CC +You can quickly and easily deploy such a cluster by following one of the [installation guides](/getting-started/install-on-clusters/kubernetes/index). +*/} + +## Configure namespaces + +This guide will deploy pods in a Kubernetes namespace. Let's create the `Namespace` object for this guide. + +```bash +kubectl create ns policy-demo +``` + +## Create demo pods + +We'll use Kubernetes `Deployment` objects to easily create pods in the namespace. + +1. Create some nginx pods in the `policy-demo` namespace. + + ```bash + kubectl create deployment --namespace=policy-demo nginx --image=nginx + ``` + +1. Expose them through a service. + + ```bash + kubectl expose --namespace=policy-demo deployment nginx --port=80 + ``` + +1. Ensure the nginx service is accessible. + + ```bash + kubectl run --namespace=policy-demo access --rm -ti --image busybox /bin/sh + ``` + + This should open up a shell session inside the `access` pod, as shown below. + + ``` + Waiting for pod policy-demo/access-472357175-y0m47 to be running, status is Pending, pod ready: false + + If you don't see a command prompt, try pressing enter. + + / # + ``` + +1. From inside the `access` pod, attempt to reach the `nginx` service. + + ```bash + wget -q nginx -O - + ``` + + You should see a response from `nginx`. Great! Our service is accessible. You can exit the pod now. + +## Enable isolation + +Let's turn on isolation in our `policy-demo` namespace. $[prodname] will then prevent connections to pods in this namespace. + +Running the following command creates a NetworkPolicy which implements a default deny behavior for all pods in the `policy-demo` namespace. + +```bash +kubectl create -f - < diff --git a/calico-cloud_versioned_docs/version-24-1/users/create-and-assign-custom-roles.mdx b/calico-cloud_versioned_docs/version-24-1/users/create-and-assign-custom-roles.mdx new file mode 100644 index 0000000000..36db636497 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/users/create-and-assign-custom-roles.mdx @@ -0,0 +1,85 @@ +--- +description: Create and assign Calico Cloud custom roles to grant cluster-scoped, tier-scoped, or namespace-scoped permissions instead of broad predefined global roles. +--- + +import IconUser from '/img/icons/user-icon.svg'; + +# Creating and assigning custom roles + +As an administrator, you can create custom, cluster-specific roles that restrict users to particular functions on a cluster. + +## Overview + +$[prodname] comes with a set of predefined global roles that let you give permissions to users based on what they need to do. +For example, a user with the Security global role has a broader set of permissions than a user with the Viewer global role. +These permissions apply to all clusters. + +But in some cases these global roles can be too broad. + +By creating and assigning custom roles, you can be much more discriminating about what permissions you give users. +For example, you could create a role that allows the user to modify network policy for a particular tier and namespace and gives view access to all other network policies. +Or you can add permissions for a user to view dashboards that show data from only one namespace in a cluster. +Permissions are assigned on a cluster-by-cluster basis. + +## Required permissions for common $[prodname] features + +Certain permissions are required for a user to access common $[prodname] features. + +| Feature area | Required permissions | Notes | +| --| -- | -- | +| Alerts | • **View Alerts**
and
•**View All Logs** | | +| Dashboard | • **View All Logs**
and
• **View Global Network Sets** or **View Network Sets** | These permissions are required for the dashboard to fully populate. All users are granted limited dashboard metrics by having access to a cluster. Users with **View Network Sets** have access to dashboards that show data only from namespaces that they have been explicitly given access to. | +| Network policies | • **View** or **Modify Policies**
or
• **View** or **Modify Global Policies**
and (optional)
• **View Audit Logs** or **View All Logs** | The **Policies** permissions apply to one or more namespaces. The **Global Policies** permissions apply to the whole cluster. These permissions are also scoped by [policy tier](../network-policy/policy-tiers/tiered-policy.mdx).

The optional **View Audit Logs** or **View All Logs** let users view the change history on the policies. | +| Service graph | • **View All Logs**
and
• **View** or **Modify Network Sets**
and (optional)
• **View** or **Modify Packet Captures** | Network sets can be restricted to a namespace or set to all namespaces to see all flows. | +| Threat feeds | • **View** or **Modify Threat Feeds** | | +| Timeline | • **View Event Logs** or **View All Logs** | | + +## Before you begin + +* You are signed in with owner or administrator permissions to the web console. + +## Create a custom role and add permissions + +1. Click the user icon > **Manage Team**. +1. Under the **Roles** tab, click **Add Role**, enter a name and description for the custom role, and then click **Save**. +1. Select the cluster you want to want the role to apply to by clicking **Cluster:** and choosing the cluster. +1. Locate your new role in the list, select **Action** > **Manage permissions** > **Edit**, and the click **Add Permission**. +1. Under **Permission**, choose a permission type from the list. + Depending on the permission, you may also need to choose a namespace or policy tier. +1. (optional) Click **Add permission** to add more permissions to your role for this cluster. +1. Click **Save** to save these permissions to the role for this cluster. +1. (optional) If you want to add the permissions for another cluster, repeat steps 3 to 7 for the cluster. + +## Assign custom roles to a user + +1. Select the user icon > **Manage Team**. +1. Under the **Users** tab, locate the user in the list and select **Action** > **Edit**. +1. Select the checkboxes for each custom role you want to assign to this user and then click **Save**. + +## Export custom roles and apply to other managed clusters + +You can export custom roles from one cluster and apply them to another cluster. + +:::note +Importing custom roles is fully supported on managed clusters running Calico Cloud 18.3 or higher. +::: + +***Prerequisites*** + +* You connected two or more managed clusters to Calico Cloud. +* You have a managed cluster with one or more custom roles. +* You have `kubectl` administrator permissions for the managed clusters you want to apply custom roles to. + +***Procedure*** + +1. From the cluster menu in the web console, select the managed cluster that has the custom roles you want to export. +1. Click the user icon > **Manage Team**. +1. Under the **Roles** tab, click **Export Custom Roles** and select **Download YAML** to download the custom role definitions. +This file contains definitions for all the custom roles you created in this cluster. +1. For each managed cluster you want to apply the custom roles to, run the following command: + + ```bash + kubectl apply -f roles..yaml + ``` + + The custom roles are available immediately for this cluster in the web console. \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/users/create-custom-role-for-entra-id-group.mdx b/calico-cloud_versioned_docs/version-24-1/users/create-custom-role-for-entra-id-group.mdx new file mode 100644 index 0000000000..2c03de90a7 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/users/create-custom-role-for-entra-id-group.mdx @@ -0,0 +1,54 @@ +--- +description: Assign Calico Cloud roles to a Microsoft Entra ID security group so role-based access is managed through identity-provider group membership. +title: Assign roles to Entra ID groups +--- + +import IconUser from '/img/icons/user-icon.svg'; + +# Give role-based access to an Entra ID group + +If you have Microsoft Entra ID configured as your identity provider, you can define role-based access in Calico Cloud and assign roles to an Entra ID (formerly Azure AD) security group. +By managing membership in that security group, you can manage role-based access to Calico Cloud directly from your identity provider portal. + +## Limitations + +The following organization-level permissions cannot be assigned through IdP groups: + +- Manage Team +- Usage Metrics + +To grant these permissions, assign them directly to individual users in the Calico Cloud web console. + +## Prerequisites + +* You are signed in to the web console with owner or administrator permissions. +* Entra ID is set up as an identity provider for Calico Cloud. + To set up an identity provider for Calico Cloud, open a [support ticket](https://support.tigera.io). +* You have administrator permissions for your organization in the Azure Portal. +* You have the Object ID for an Entra ID security group. +* The **Email** property for all users in the security group has a valid email address. + +## Add an Entra ID security group and assign role-based permissions for Calico Cloud + +1. In the web console, click the user icon > **Manage Team**. +1. Select the **Roles** tab, click the **Add IdP Group** button, and then enter the following information: + * **IdP Group Name**: Enter a name for the group that will be displayed in the web console. + * **IdP Group Identifier**: Enter the Object ID for the Entra ID security group. + * (Optional) **Description**: Enter a description for the group. + :::note + If you don't see an **Add IdP Group** button, your organization does not have an identity provider configured. + To set up an identity provider for Calico Cloud, open a [support ticket](https://support.tigera.io). + ::: +2. Select one or more predefined user roles from the **Predefined Roles** list to assign to this group. +1. Click **Save**. + +## Add custom permissions to an IdP group + +1. In the web console, click the user icon > **Manage Team**. +1. Select the **Roles** tab, and then locate the role you created for the Entra ID group. +1. Click **Actions > Manage permissions**. +1. To add permissions, locate your new IdP group role under the **Roles** tab, select **Action > Manage permissions > Edit**, and then click **Add Permission**. +1. Under **Permission**, choose a permission type from the list. + Depending on the permission, you may also need to choose a namespace or policy tier. +1. Optional: To add more permissions to your role, click **Add permission** again and repeat the previous step. +1. Click **Save** to save these permissions to the role for this cluster. \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/users/index.mdx b/calico-cloud_versioned_docs/version-24-1/users/index.mdx new file mode 100644 index 0000000000..39ea40ade9 --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/users/index.mdx @@ -0,0 +1,17 @@ +--- +description: Reference index for Calico Cloud user management, custom roles, identity-provider group bindings, and predefined web console permissions. +title: Manage users and user permissions for the web console. +hide_table_of_contents: true +--- + +import { DocCardLink, DocCardLinkLayout } from '/src/___new___/components'; + +# Users + +Manage users and user permissions for the web console. + + + + + + diff --git a/calico-cloud_versioned_docs/version-24-1/users/user-management.mdx b/calico-cloud_versioned_docs/version-24-1/users/user-management.mdx new file mode 100644 index 0000000000..c6a15d06bb --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/users/user-management.mdx @@ -0,0 +1,208 @@ +--- +description: Reference for the Calico Cloud predefined user roles and authentication options, including Owner, Admin, Security, Viewer, and Google social login. +--- + +# Set up users + +## Authentication + +$[prodname] supports Google Social login and username / password for user authentication. + +## Roles and authorization + +Users can have one or more of the following predefined user roles to access features in the web console. The default permissions align with typical needs for each role. + +### Owner + +The Owner role has the highest level of access and typically corresponds to the account creator. + +:::note +The Owner role cannot be assigned to new users. The only Owner is the user who created the $[prodname] account. +::: + +| Feature | Permission Level | +| :------------------------------ | :----------------- | +| Service Graph and Flow Visualizer | view | +| Policies | view, edit | +| Nodes and Endpoints | view | +| Network Sets | view, edit | +| Managed Clusters | view, edit, delete | +| Timeline | view | +| Alerts | view, edit | +| Manage Team | view, edit | +| Usage Metrics | view | +| Threat Feeds | view, edit | +| Web Application Firewall | view, edit | +| Dashboards | view, edit | + +### Admin + +The Admin role provides broad administrative access for day-to-day configuration and management of $[prodname]. + +| Feature | Permission Level | +| :------------------------------ | :----------------- | +| Service Graph and Flow Visualizer | view | +| Policies | view, edit | +| Nodes and Endpoints | view | +| Network Sets | view, edit | +| Managed Clusters | view, edit, delete | +| Timeline | view | +| Alerts | view, edit | +| Manage Team | view, edit | +| Usage Metrics | - | +| Threat Feeds | view, edit | +| Web Application Firewall | view, edit | +| Dashboards | view, edit | + +### User Admin + +The User Admin role has the ability to manage team members and their assigned roles. + +| Feature | Permission Level | +| :------------------------------ | :----------------- | +| Service Graph and Flow Visualizer | - | +| Policies | - | +| Nodes and Endpoints | - | +| Network Sets | - | +| Managed Clusters | - | +| Timeline | - | +| Alerts | - | +| Manage Team | view, edit | +| Usage Metrics | - | +| Threat Feeds | - | +| Web Application Firewall | - | +| Dashboards | - | + +### Cluster Connection Admin + +The Cluster Connection Admin role has administrative capabilities of managed clusters. + +| Feature | Permission Level | +| :------------------------------ | :----------------- | +| Service Graph and Flow Visualizer | - | +| Policies | - | +| Nodes and Endpoints | - | +| Network Sets | - | +| Managed Clusters | view, edit, delete | +| Timeline | - | +| Alerts | - | +| Manage Team | - | +| Usage Metrics | - | +| Threat Feeds | - | +| Web Application Firewall | - | +| Dashboards | - | + +### Viewer + +The Viewer role provides read-only access to most operational and configuration data within $[prodname]. Ideal for users who need visibility without making changes. + +| Feature | Permission Level | +| :------------------------------ | :--------------- | +| Service Graph and Flow Visualizer | view | +| Policies | view | +| Nodes and Endpoints | view | +| Network Sets | view | +| Managed Clusters | view | +| Timeline | view | +| Alerts | view | +| Manage Team | view | +| Usage Metrics | - | +| Threat Feeds | view | +| Web Application Firewall | view | +| Dashboards | view | + +### DevOps + +The DevOps role is designed for users responsible for application deployment, CI/CD integration, and managing network policies and configurations relevant to their applications. + +| Feature | Permission Level | +| :------------------------------ | :--------------- | +| Service Graph and Flow Visualizer | view | +| Policies | view, edit | +| Nodes and Endpoints | view | +| Network Sets | view, edit | +| Managed Clusters | view, edit | +| Timeline | view | +| Alerts | view, edit | +| Manage Team | view | +| Usage Metrics | - | +| Threat Feeds | view, edit | +| Web Application Firewall | view | +| Dashboards | view | + +### Security + +The Security role focuses on security posture management, including policy definition, threat monitoring, and incident response. + +| Feature | Permission Level | +| :------------------------------ | :--------------- | +| Service Graph and Flow Visualizer | view | +| Policies | view, edit | +| Nodes and Endpoints | view | +| Network Sets | view, edit | +| Managed Clusters | view | +| Timeline | view | +| Alerts | view, edit | +| Manage Team | view | +| Usage Metrics | - | +| Threat Feeds | view, edit | +| Web Application Firewall | view, edit | +| Dashboards | view | + +### Usage Metrics + +This role grants specific access to view usage metrics for the $[prodname] account. + +| Feature | Permission Level | +| :------------------------------ | :--------------- | +| Service Graph and Flow Visualizer | - | +| Policies | - | +| Nodes and Endpoints | - | +| Network Sets | - | +| Managed Clusters | - | +| Timeline | - | +| Alerts | - | +| Manage Team | - | +| Usage Metrics | view | +| Threat Feeds | - | +| Web Application Firewall | - | +| Dashboards | - | + +### Dashboards Admin + +This role grants administrative permissions specifically for creating, managing, and sharing custom dashboards within $[prodname]. + +| Feature | Permission Level | +| :------------------------------ | :--------------- | +| Service Graph and Flow Visualizer | - | +| Policies | - | +| Nodes and Endpoints | - | +| Network Sets | - | +| Managed Clusters | - | +| Timeline | - | +| Alerts | - | +| Manage Team | - | +| Usage Metrics | - | +| Threat Feeds | - | +| Web Application Firewall | - | +| Dashboards | view, edit | + +## Add your own identity provider + +$[prodname] works with any identity provider that supports [OpenID Connect](https://openid.net/connect/). For example, OKTA, Google, and Azure AD. + +To add an identity provider, open a [Support ticket](https://support.tigera.io/). + +### Azure AD requirements + +To add Azure AD as your identity provider, create an Active Directory "App Registration" with a Redirect URI of type "Web" set to https://auth.calicocloud.io/login/callback. + +Enable "ID Token" for implicit flows. + +Add the following Microsoft Graph API delegated permissions: + +- User.Read +- OpenId permissions: + - email + - openid + - profile \ No newline at end of file diff --git a/calico-cloud_versioned_docs/version-24-1/variables.js b/calico-cloud_versioned_docs/version-24-1/variables.js new file mode 100644 index 0000000000..1335b4572b --- /dev/null +++ b/calico-cloud_versioned_docs/version-24-1/variables.js @@ -0,0 +1,32 @@ +const releases = require('./releases.json'); + +const variables = { + releaseTitle: 'v3.24.0-1.0', + cloudUserVersion: 'v24.0.0', + prodname: 'Calico Cloud', + manifestsUrl: 'https://raw.githubusercontent.com/projectcalico/calico/v3.32.0', + prodnamedash: 'calico-cloud', + baseUrl: '/calico-cloud', + filesUrl: 'https://docs.calicocloud.io', + filesUrl_CE: 'https://downloads.tigera.io/ee/v3.24.0-1.0', + tutorialFilesURL: 'https://docs.tigera.io/files', + prodnameWindows: 'Calico Enterprise for Windows', + rootDirWindows: 'C:\\TigeraCalico', + nodecontainer: 'cnx-node', + noderunning: 'calico-node', + cloudversion: 'v3.24.0-1.0-4', + clouddownloadurl: 'https://installer.calicocloud.io/manifests/v3.24.0-1.0-4', + clouddownloadbase: 'https://installer.calicocloud.io', + cloudoperatorimage: 'quay.io/tigera/cc-operator', + tigeraOperator: releases[0]['tigera-operator'], + dikastesVersion: releases[0].components.dikastes.version, + releases, + registry: 'quay.io/', + envoyVersion: '1.8.2', + imageNames: { + node: 'tigera/cnx-node', + kubeControllers: 'tigera/kube-controllers', + }, +}; + +module.exports = variables; diff --git a/calico-cloud_versioned_sidebars/version-24-1-sidebars.json b/calico-cloud_versioned_sidebars/version-24-1-sidebars.json new file mode 100644 index 0000000000..6d28d3cc3c --- /dev/null +++ b/calico-cloud_versioned_sidebars/version-24-1-sidebars.json @@ -0,0 +1,771 @@ +{ + "calicoCloudSidebar": [ + { + "type": "category", + "label": "About Calico", + "link": { + "type": "doc", + "id": "about/index" + }, + "items": [ + "about/calico-product-editions" + ] + }, + { + "type": "category", + "label": "Calico Cloud Free Tier", + "link": { + "type": "doc", + "id": "free/index" + }, + "items": [ + "free/overview", + "free/quickstart", + "free/connect-cluster-free", + "free/disconnect-cluster-free" + ] + }, + { + "type": "category", + "label": "Install and upgrade", + "link": { + "type": "doc", + "id": "get-started/index" + }, + "items": [ + "get-started/cc-arch-diagram", + "get-started/connect-cluster", + "get-started/system-requirements", + "get-started/prepare-cluster", + "get-started/install-cluster", + "get-started/install-private-registry", + "get-started/install-automated", + "get-started/setup-private-registry", + "get-started/windows-limitations", + "get-started/checklist", + "get-started/operator-checklist", + "get-started/upgrade-cluster" + ] + }, + { + "type": "category", + "label": "Users", + "link": { + "type": "doc", + "id": "users/index" + }, + "items": [ + "users/user-management", + "users/create-and-assign-custom-roles", + "users/create-custom-role-for-entra-id-group" + ] + }, + { + "type": "category", + "label": "Tutorials", + "link": { + "type": "doc", + "id": "tutorials/index" + }, + "items": [ + { + "type": "category", + "label": "Calico Cloud features", + "link": { + "type": "doc", + "id": "tutorials/calico-cloud-features/index" + }, + "items": [ + "tutorials/calico-cloud-features/tour", + "tutorials/calico-cloud-features/service-graph", + "tutorials/calico-cloud-features/networksets", + "tutorials/calico-cloud-features/projects" + ] + }, + { + "type": "category", + "label": "Secure ingress and egress for applications", + "link": { + "type": "doc", + "id": "tutorials/applications/index" + }, + "items": [ + "tutorials/applications/ingress-microservices", + "tutorials/applications/egress-controls" + ] + }, + { + "type": "category", + "label": "Implement enterprise security controls", + "link": { + "type": "doc", + "id": "tutorials/enterprise-security/index" + }, + "items": [ + "tutorials/enterprise-security/namespace-isolation", + "tutorials/enterprise-security/global-egress", + "tutorials/enterprise-security/default-deny", + "tutorials/enterprise-security/platform" + ] + }, + { + "type": "category", + "label": "Kubernetes networking for beginners", + "link": { + "type": "doc", + "id": "tutorials/training/index" + }, + "items": [ + "tutorials/training/about-network-policy", + "tutorials/training/about-kubernetes-services", + "tutorials/training/about-kubernetes-ingress", + "tutorials/training/about-kubernetes-egress" + ] + }, + { + "type": "category", + "label": "Kubernetes tutorials and demos", + "link": { + "type": "doc", + "id": "tutorials/kubernetes-tutorials/index" + }, + "items": [ + "tutorials/kubernetes-tutorials/kubernetes-demo", + "tutorials/kubernetes-tutorials/kubernetes-network-policy", + "tutorials/kubernetes-tutorials/kubernetes-policy-basic", + "tutorials/kubernetes-tutorials/kubernetes-policy-advanced" + ] + } + ] + }, + { + "type": "category", + "label": "Network policy", + "link": { + "type": "doc", + "id": "network-policy/index" + }, + "items": [ + { + "type": "category", + "label": "Policy recommendations", + "link": { + "type": "doc", + "id": "network-policy/recommendations/index" + }, + "items": [ + "network-policy/recommendations/policy-recommendations", + "network-policy/recommendations/learn-about-policy-recommendations" + ] + }, + "network-policy/policy-best-practices", + { + "type": "category", + "label": "Tiered network policy", + "link": { + "type": "doc", + "id": "network-policy/policy-tiers/index" + }, + "items": [ + "network-policy/policy-tiers/tiered-policy", + "network-policy/policy-tiers/policy-tutorial-ui", + "network-policy/policy-tiers/calico-system", + "network-policy/policy-tiers/rbac-tiered-policies" + ] + }, + "network-policy/networksets", + "network-policy/default-deny", + "network-policy/staged-network-policies", + "network-policy/policy-troubleshooting", + { + "type": "category", + "label": "Calico Cloud network policy for beginners", + "link": { + "type": "doc", + "id": "network-policy/beginners/index" + }, + "items": [ + "network-policy/beginners/kubernetes-default-deny", + "network-policy/beginners/calico-network-policy", + "network-policy/beginners/calico-labels", + "network-policy/beginners/simple-policy-cnx", + { + "type": "category", + "label": "Policy rules", + "link": { + "type": "doc", + "id": "network-policy/beginners/policy-rules/index" + }, + "items": [ + "network-policy/beginners/policy-rules/policy-rules-overview", + "network-policy/beginners/policy-rules/namespace-policy", + "network-policy/beginners/policy-rules/service-accounts", + "network-policy/beginners/policy-rules/service-policy", + "network-policy/beginners/policy-rules/external-ips-policy", + "network-policy/beginners/policy-rules/icmp-ping" + ] + }, + { + "type": "category", + "label": "Policy for services", + "link": { + "type": "doc", + "id": "network-policy/beginners/services/index" + }, + "items": [ + "network-policy/beginners/services/kubernetes-node-ports", + "network-policy/beginners/services/services-cluster-ips" + ] + } + ] + }, + "network-policy/domain-based-policy", + { + "type": "category", + "label": "Application layer policies", + "link": { + "type": "doc", + "id": "network-policy/application-layer-policies/index" + }, + "items": [ + "network-policy/application-layer-policies/alp", + "network-policy/application-layer-policies/alp-tutorial" + ] + }, + { + "type": "category", + "label": "Policy for firewalls", + "link": { + "type": "doc", + "id": "network-policy/policy-firewalls/index" + }, + "items": [ + { + "type": "category", + "label": "Fortinet firewall integrations", + "link": { + "type": "doc", + "id": "network-policy/policy-firewalls/fortinet-integration/index" + }, + "items": [ + "network-policy/policy-firewalls/fortinet-integration/overview", + "network-policy/policy-firewalls/fortinet-integration/firewall-integration", + "network-policy/policy-firewalls/fortinet-integration/fortimgr-integration" + ] + } + ] + }, + "network-policy/hosts/kubernetes-nodes", + "network-policy/hosts/host-forwarded-traffic", + { + "type": "category", + "label": "Policy for extreme traffic", + "link": { + "type": "doc", + "id": "network-policy/extreme-traffic/index" + }, + "items": [ + "network-policy/extreme-traffic/high-connection-workloads", + "network-policy/extreme-traffic/defend-dos-attack" + ] + } + ] + }, + { + "type": "category", + "label": "Observability", + "link": { + "type": "doc", + "id": "observability/index" + }, + "items": [ + "observability/alerts", + "observability/dashboards", + "observability/create-custom-dashboard", + "observability/packetcapture", + "observability/visualize-traffic", + { + "type": "category", + "label": "Calico Cloud logs", + "link": { + "type": "doc", + "id": "observability/elastic/index" + }, + "items": [ + "observability/elastic/overview", + "observability/elastic/archive-storage", + "observability/elastic/aggregation", + { + "type": "category", + "label": "Flow logs", + "link": { + "type": "doc", + "id": "observability/elastic/flow/index" + }, + "items": [ + "observability/elastic/flow/datatypes", + "observability/elastic/flow/filtering", + "observability/elastic/flow/aggregation", + "observability/elastic/flow/hep", + "observability/elastic/flow/tcpstats", + "observability/elastic/flow/processpath" + ] + }, + "observability/elastic/audit-overview", + { + "type": "category", + "label": "DNS logs", + "link": { + "type": "doc", + "id": "observability/elastic/dns/index" + }, + "items": [ + "observability/elastic/dns/dns-logs", + "observability/elastic/dns/filtering-dns" + ] + }, + "observability/elastic/bgp", + { + "type": "category", + "label": "L7 logs", + "link": { + "type": "doc", + "id": "observability/elastic/l7/index" + }, + "items": [ + { + "type": "doc", + "id": "observability/elastic/l7/overview", + "label": "Overview" + }, + { + "type": "doc", + "id": "observability/elastic/l7/enable-ebpf-collector", + "label": "Enable eBPF collector" + }, + { + "type": "doc", + "id": "observability/elastic/l7/enable-waypoint-collector", + "label": "Enable Waypoint collector" + }, + { + "type": "doc", + "id": "observability/elastic/l7/configure", + "label": "Enable Envoy collector" + }, + { + "type": "doc", + "id": "observability/elastic/l7/datatypes", + "label": "Data types" + } + ] + } + ] + }, + "observability/kube-audit", + "observability/iptables" + ] + }, + { + "type": "category", + "label": "Threat defense", + "link": { + "type": "doc", + "id": "threat/index" + }, + "items": [ + "threat/configuring-webhooks", + "threat/security-event-management", + "threat/suspicious-ips", + "threat/suspicious-domains", + "threat/tor-vpn-feed-and-dashboard", + "threat/deeppacketinspection", + "threat/web-application-firewall", + "threat/deploying-waf-ingress-gateway", + { + "type": "category", + "label": "Gateway WAF", + "items": [ + "threat/gateway-waf/overview", + "threat/gateway-waf/tutorial", + "threat/gateway-waf/enable", + "threat/gateway-waf/set-a-baseline", + "threat/gateway-waf/attach-waf-to-a-route", + "threat/gateway-waf/write-custom-rules", + "threat/gateway-waf/validate-config", + "threat/gateway-waf/multi-tenant-setup", + "threat/gateway-waf/troubleshooting", + "threat/gateway-waf/reference" + ] + } + ] + }, + { + "type": "category", + "label": "Security", + "link": { + "type": "doc", + "id": "compliance/index" + }, + "items": [ + { + "type": "category", + "label": "Istio Ambient Mode", + "link": null, + "items": [ + "compliance/istio/about-istio-ambient", + "compliance/istio/deploy-istio-ambient" + ] + }, + "compliance/encrypt-cluster-pod-traffic", + "compliance/configure-http-proxy" + ] + }, + { + "type": "category", + "label": "Networking", + "link": { + "type": "doc", + "id": "networking/index" + }, + "items": [ + { + "type": "category", + "label": "Ingress gateway", + "link": null, + "items": [ + "networking/ingress-gateway/about-calico-ingress-gateway", + "networking/ingress-gateway/create-ingress-gateway", + "networking/ingress-gateway/customize-ingress-gateway", + "networking/ingress-gateway/tutorial-ingress-gateway-canary" + ] + }, + { + "type": "category", + "label": "Networking basics", + "link": { + "type": "doc", + "id": "networking/training/index" + }, + "items": [ + "networking/training/about-networking", + "networking/training/about-kubernetes-networking" + ] + }, + { + "type": "category", + "label": "Configure Calico Cloud networking", + "link": { + "type": "doc", + "id": "networking/configuring/index" + }, + "items": [ + "networking/configuring/bgp", + "networking/configuring/bgp-to-workload", + "networking/configuring/dual-tor", + "networking/configuring/multiple-networks", + "networking/configuring/vxlan-ipip", + "networking/configuring/advertise-service-ips", + "networking/configuring/local-subnet-l2-reachability", + "networking/configuring/mtu", + "networking/configuring/custom-bgp-config", + "networking/configuring/secure-bgp", + "networking/configuring/workloads-outside-cluster", + "networking/configuring/pod-mac-address", + "networking/configuring/allowed-source-prefixes", + "networking/configuring/node-local-dns-cache", + "networking/configuring/qos-controls", + "networking/configuring/add-maglev-load-balancing", + "networking/configuring/mark-lb-node-for-maintenance" + ] + }, + { + "type": "category", + "label": "Egress gateways", + "link": { + "type": "doc", + "id": "networking/egress/index" + }, + "items": [ + "networking/egress/egress-gateway-on-prem", + "networking/egress/egress-gateway-host-ip", + "networking/egress/egress-gateway-aws", + "networking/egress/egress-gateway-azure", + "networking/egress/egress-gateway-maintenance", + "networking/egress/external-network", + "networking/egress/troubleshoot" + ] + }, + { + "type": "category", + "label": "Customize IP address management", + "link": { + "type": "doc", + "id": "networking/ipam/index" + }, + "items": [ + "networking/ipam/get-started-ip-addresses", + "networking/ipam/initial-ippool", + "networking/ipam/ip-autodetection", + "networking/ipam/ipv6", + "networking/ipam/use-specific-ip", + "networking/ipam/assign-ip-addresses-topology", + "networking/ipam/migrate-pools", + "networking/ipam/change-block-size", + "networking/ipam/legacy-firewalls", + "networking/ipam/service-loadbalancer" + ] + } + ] + }, + { + "type": "category", + "label": "Cluster mesh", + "link": { + "type": "doc", + "id": "multicluster/index" + }, + "items": [ + "multicluster/overview", + "multicluster/kubeconfig", + "multicluster/services-controller", + "multicluster/aws" + ] + }, + { + "type": "category", + "label": "Operations", + "link": { + "type": "doc", + "id": "operations/index" + }, + "items": [ + "operations/cluster-management", + "operations/disconnect", + "operations/usage-metrics", + "operations/usage-api", + "operations/comms/index", + { + "type": "category", + "label": "Monitoring", + "link": { + "type": "doc", + "id": "operations/monitor/index" + }, + "items": [ + { + "type": "category", + "label": "Prometheus", + "link": { + "type": "doc", + "id": "operations/monitor/prometheus/index" + }, + "items": [ + "operations/monitor/prometheus/support", + "operations/monitor/metrics/recommended-metrics", + "operations/monitor/prometheus/byo-prometheus", + "operations/monitor/prometheus/configure-prometheus", + "operations/monitor/prometheus/alertmanager" + ] + }, + { + "type": "category", + "label": "Metrics", + "link": { + "type": "doc", + "id": "operations/monitor/metrics/index" + }, + "items": [ + "operations/monitor/metrics/operator-metrics", + "operations/monitor/metrics/bgp-metrics", + "operations/monitor/metrics/policy-metrics", + "operations/monitor/metrics/elasticsearch-and-fluentd-metrics" + ] + } + ] + }, + { + "type": "category", + "label": "eBPF", + "link": { + "type": "doc", + "id": "operations/ebpf/index" + }, + "items": [ + "operations/ebpf/use-cases-ebpf", + "operations/ebpf/enabling-ebpf", + "operations/ebpf/troubleshoot-ebpf" + ] + }, + "operations/component-logs" + ] + }, + { + "type": "category", + "label": "Reference", + "link": { + "type": "doc", + "id": "reference/index" + }, + "items": [ + "reference/api", + "reference/installation/api", + { + "type": "category", + "label": "Resource definitions", + "link": { + "type": "doc", + "id": "reference/resources/index" + }, + "items": [ + "reference/resources/overview", + "reference/resources/alertexception", + "reference/resources/bfdconfig", + "reference/resources/bgpconfig", + "reference/resources/bgppeer", + "reference/resources/bgpfilter", + "reference/resources/blockaffinity", + "reference/resources/caliconodestatus", + "reference/resources/deeppacketinspection", + "reference/resources/felixconfig", + "reference/resources/egressgatewaypolicy", + "reference/resources/globalalert", + "reference/resources/globalnetworkpolicy", + "reference/resources/globalnetworkset", + "reference/resources/globalthreatfeed", + "reference/resources/hostendpoint", + "reference/resources/ippool", + "reference/resources/ipreservation", + "reference/resources/ipamconfig", + "reference/resources/licensekey", + "reference/resources/kubecontrollersconfig", + "reference/resources/managedcluster", + "reference/resources/networkpolicy", + "reference/resources/networkset", + "reference/resources/node", + "reference/resources/packetcapture", + "reference/resources/policyrecommendations", + "reference/resources/remoteclusterconfiguration", + "reference/resources/securityeventwebhook", + "reference/resources/stagedglobalnetworkpolicy", + "reference/resources/stagedkubernetesnetworkpolicy", + "reference/resources/stagednetworkpolicy", + "reference/resources/tier", + "reference/resources/workloadendpoint" + ] + }, + { + "type": "category", + "label": "Architecture and network design", + "link": { + "type": "doc", + "id": "reference/architecture/index" + }, + "items": [ + "reference/architecture/data-path", + { + "type": "category", + "label": "Network design", + "link": { + "type": "doc", + "id": "reference/architecture/design/index" + }, + "items": [ + "reference/architecture/design/l2-interconnect-fabric", + "reference/architecture/design/l3-interconnect-fabric" + ] + } + ] + }, + { + "type": "category", + "label": "Component resources", + "link": { + "type": "doc", + "id": "reference/component-resources/index" + }, + "items": [ + "reference/component-resources/configuration", + "reference/component-resources/configure-resources", + { + "type": "category", + "label": "Calico Cloud Kubernetes controllers", + "link": { + "type": "doc", + "id": "reference/component-resources/kube-controllers/index" + }, + "items": [ + "reference/component-resources/kube-controllers/prometheus" + ] + }, + { + "type": "category", + "label": "Calico Cloud node (cnx-node)", + "link": { + "type": "doc", + "id": "reference/component-resources/node/index" + }, + "items": [ + "reference/component-resources/node/configuration", + { + "type": "category", + "label": "Felix", + "link": { + "type": "doc", + "id": "reference/component-resources/node/felix/index" + }, + "items": [ + "reference/component-resources/node/felix/configuration", + "reference/component-resources/node/felix/prometheus" + ] + } + ] + } + ] + }, + { + "type": "category", + "label": "Configuration on public clouds", + "link": { + "type": "doc", + "id": "reference/public-cloud/index" + }, + "items": [ + "reference/public-cloud/aws", + "reference/public-cloud/azure", + "reference/public-cloud/gce" + ] + }, + { + "type": "category", + "label": "Host endpoints", + "link": { + "type": "doc", + "id": "reference/host-endpoints/index" + }, + "items": [ + "reference/host-endpoints/overview", + "reference/host-endpoints/connectivity", + "reference/host-endpoints/objects", + "reference/host-endpoints/selector", + "reference/host-endpoints/failsafe", + "reference/host-endpoints/pre-dnat", + "reference/host-endpoints/forwarded", + "reference/host-endpoints/summary", + "reference/host-endpoints/conntrack" + ] + }, + "reference/attribution", + "reference/rest-api-reference", + "reference/faq", + "reference/feature-lifecycle-reference" + ] + }, + "get-help/support", + { + "type": "doc", + "label": "Release notes", + "id": "release-notes/index" + } + ] +} diff --git a/calico-cloud_versions.json b/calico-cloud_versions.json index e60323d20a..f008c8a905 100644 --- a/calico-cloud_versions.json +++ b/calico-cloud_versions.json @@ -1,3 +1,4 @@ [ + "24-1", "23-2" ]