diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..211cc725e --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,11 @@ +FROM node:21-alpine3.19 +RUN apk add --no-cache \ + git \ + nano\ + openssh \ + py3-pip +ENV USER node +USER $USER +# python packages (as mkdocs) are installed in the user's home directory +# but we need them to be accessible from $PATH +ENV PATH="$PATH:/home/$USER/.local/bin" diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..d96d2657e --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,26 @@ +{ + "name": "Pi-hole doc devcontainer", + "dockerFile": "Dockerfile", + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "sh .devcontainer/post-create.sh", + "customizations": { + // Configure properties specific to VS Code. + "vscode": { + "settings": {}, + "extensions": [ + "eamodio.gitlens", + "EditorConfig.EditorConfig", + "github.vscode-github-actions", + "DavidAnson.vscode-markdownlint" + + ] + } + }, + "containerEnv": { + "GIT_EDITOR": "nano" + }, + "mounts": [ + "type=bind,source=/home/${localEnv:USER}/.ssh,target=/home/node/.ssh,readonly" + ], + "forwardPorts": [8000] +} diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh new file mode 100644 index 000000000..44c9b6bb6 --- /dev/null +++ b/.devcontainer/post-create.sh @@ -0,0 +1,3 @@ +#! /bin/sh +npm install +pip3 install -r requirements.txt --break-system-packages --no-warn-script-location diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 510342836..9726936c7 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -16,4 +16,4 @@ jobs: uses: codespell-project/actions-codespell@master with: ignore_words_file: .codespellignore - skip: ./docs/routers/fritzbox-de.md,./mkdocs.yml,./package.json,./package-lock.json,./.markdownlint.json,./requirements.txt + skip: ./docs/routers/fritzbox-de.md,./mkdocs.yml,./package.json,./package-lock.json,./.markdownlint.json,./requirements.txt, ./MathJax-es5/* diff --git a/docs/abbreviations.md b/docs/abbreviations.md index d4af635b7..af4e99c2c 100644 --- a/docs/abbreviations.md +++ b/docs/abbreviations.md @@ -1,15 +1,18 @@ +*[2FA]: Two-factor authentication +*[API]: Application Programming Interface (a set of subroutine definitions, protocols, and tools for building application software) +*[CSRF]: Cross-site request forgery *[DNS]: Domain Name Service (decentralized naming system for computers, services, or other resources connected to the Internet) -*[FTL]: Pi-hole's Faster Than Light daemon +*[DnyDNS]: Dynamic DNS record pointing to a frequently changing IP address *[DHCP]: Dynamic Host Configuration Protocol (network management protocol for configuring Internet Protocol version 4 (IPv4) hosts with IP addresses) *[DHCPv6]: Dynamic Host Configuration Protocol version 6 (a network protocol for configuring Internet Protocol version 6 (IPv6) hosts with IP addresses) +*[FTL]: Pi-hole's Faster Than Light daemon *[IPv4]: Internet Protocol version 4 (addresses like 192.168.0.1) *[IPv6]: Internet Protocol version 6 (addresses like 2001:db8::ff00:42:8329) *[HTTP]: Hypertext Transfer Protocol (HTTP), an application protocol for distributed, collaborative, and hypermedia information systems *[HTTPS]: HTTP Secure (HTTPS), an extension of the Hypertext Transfer Protocol (HTTP) for secure communication over a computer network *[TCP]: Transmission Control Protocol (protocol providing reliable, ordered, and error-checked delivery of data between applications running on hosts communicating via an IP network) *[UDP]: User Datagram Protocol (a network communications method for sending messages as datagrams) -*[API]: Application Programming Interface (a set of subroutine definitions, protocols, and tools for building application software) *[PE]: Privacy Extension *[PID]: Process identifier (a number used to identify a process) *[HOSTS]: The computer file /etc/hosts is an operating system file that maps hostnames to IP addresses @@ -34,9 +37,14 @@ *[Regex]: Regular expression *[regex]: Regular expression *[SQLite3]: Database engine that handles SQL databases in a file +*[SID]: Session ID *[ID]: Identifier *[SSH]: Secure Shell is a cryptographic network protocol for operating network services securely over an unsecured network *[TFTP]: Trivial File Transfer Protocol is a simple lockstep File Transfer Protocol which allows a client to get a file from or put a file onto a remote host *[TTL]: Time-To-Live is a mechanism that limits the lifespan or lifetime of data in a computer or network +*[TOTP]: Time-based One-Time Password *[NAT]: Network address translation -*[DnyDNS]: Dynamic DNS record pointing to a frequently changing IP address +*[UTF-8]: 8-bit Unicode Transformation Format - a character encoding format capable of encoding all known 1,112,064 valid Unicode characters +*[URL]: Uniform Resource Locator, commonly known as "web address" +*[REST]: Representational State Transfer - a software architecture for distributed systems like the World Wide Web (WWW) +*[XSS]: Cross-site scripting diff --git a/docs/api/auth.md b/docs/api/auth.md new file mode 100644 index 000000000..4b27410a2 --- /dev/null +++ b/docs/api/auth.md @@ -0,0 +1,421 @@ +Authentication is required for most API endpoints. The Pi-hole API uses a session-based authentication system. This means that you will not be able to use a static token to authenticate your requests. Instead, you will be given a session ID (SID) that you will have to use. If you didn't set a password for your Pi-hole, you don't have to authenticate your requests. + +To get a session ID, you will have to send a `POST` request to the `/api/auth` endpoint with a payload containing your password. Note that is also possible to use an application password instead of your regular password, e.g., if you don't want to put your password in your scripts or if you have 2FA enabled for your regular password. One application password can be generated in the web interface on the settings page. + + +???+ example "Authentication with password" + + === "bash / cURL" + + ``` bash + curl -k -X POST "https://pi.hole/api/auth" --data '{"password":"your-password"}' + ``` + + === "Python 3" + + ``` python + import requests + + url = "https://pi.hole/api/auth" + payload = {"password": "your-password"} + + response = requests.request("POST", url, json=payload, verify=False) + + print(response.text) + ``` + + === "Javascript (plain)" + + ``` javascript + var data = JSON.stringify({"password":"your-password"}); + var xhr = new XMLHttpRequest(); + + xhr.addEventListener("readystatechange", function () { + if (this.readyState === this.DONE) { + console.log(this.responseText); + } + }); + ``` + + === "Javascript (jQuery)" + + ``` javascript + $.ajax({ + url: "https://pi.hole/api/auth", + type: "POST", + data: JSON.stringify({"password":"your-password"}), + dataType: "json", + contentType: "application/json" + }).done(function(data) { + console.log(data); + }).fail(function(xhr, status, error) { + console.log(error); + }); + ``` + + === "C" + + ``` c + #include + #include + #include + + int main(void) + { + CURL *curl; + CURLcode res; + + curl_global_init(CURL_GLOBAL_ALL); + + curl = curl_easy_init(); + if(curl) { + curl_easy_setopt(curl, CURLOPT_URL, "https://pi.hole/api/auth"); + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, "{\"password\":\"your-password\"}"); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L); + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYHOST, 0L); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "POST"); + + struct curl_slist *headers = NULL; + headers = curl_slist_append(headers, "Content-Type: application/json"); + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers); + + res = curl_easy_perform(curl); + + curl_easy_cleanup(curl); + } + + curl_global_cleanup(); + + return 0; + } + ``` + + **Parameters** + + Specify either your password or an application password as `password` in the payload. + +???+ success "Success response" + + Response code: `HTTP/1.1 200 OK` + + ```json + { + "session": { + "valid": true, + "totp": false, + "sid": "vFA+EP4MQ5JJvJg+3Q2Jnw=", + "csrf": "Ux87YTIiMOf/GKCefVIOMw=", + "validity": 300 + }, + "took": 0.0002 + } + ``` + +???+ failure "Error response" + + Response code: `HTTP/1.1 401 - Unauthorized` + + ```json + { + "error": { + "key": "unauthorized", + "message": "Unauthorized", + "hint": null + }, + "took": 0.003 + } + ``` + +On success, this will return a JSON object containing the session ID (SID) and the time until your session expires (validity in seconds). +You can use this SID from this point to authenticate your requests to the API. + +## Use the SID to access API endpoints + +Once you have a valid SID, you can use it to authenticate your requests. You can do this in four different ways: + +1. In the request URI: `http://pi.hole/api/info/version?sid=9N80JpYyHRBX4c5RW95%2Fyg%3D` (your SID needs to be URL-encoded) +2. In the payload of your request: `{"sid":"vFA+EP4MQ5JJvJg+3Q2Jnw="}` +3. In the `X-FTL-SID` header: `X-FTL-SID: vFA+EP4MQ5JJvJg+3Q2Jnw=` +4. In the `sid` cookie: `Cookie: sid=vFA+EP4MQ5JJvJg+3Q2Jnw=` + +Note that when using cookie-based authentication, you will also need to send a `X-FTL-CSRF` header with the CSRF token that was returned when you authenticated. This is to prevent a certain kind of identify theft attack the Pi-hole API is immune against. + +???+ example "Authentication with SID" + + === "bash / cURL" + + ``` bash + # Example: Authentication with SID in the request URI + curl -k -X GET "https://pi.hole/api/dns/blocking?sid=vFA+EP4MQ5JJvJg+3Q2Jnw=" + ``` + + === "Python 3" + + ``` python + # Example: Authentication with SID in the request header + import requests + + url = "https://pi.hole/api/dns/blocking" + payload = {} + headers = { + "X-FTL-SID": "vFA+EP4MQ5JJvJg+3Q2Jnw=", + "X-FTL-CSRF": "Ux87YTIiMOf/GKCefVIOMw=" + } + + response = requests.request("GET", url, headers=headers, data=payload, verify=False) + + print(response.text) + ``` + + === "Javascript (plain)" + + ``` javascript + var data = null; + var xhr = new XMLHttpRequest(); + + xhr.addEventListener("readystatechange", function () { + if (this.readyState === this.DONE) { + console.log(this.responseText); + } + }); + + xhr.open("GET", "https://pi.hole/api/dns/blocking?sid=vFA+EP4MQ5JJvJg+3Q2Jnw="); + xhr.setRequestHeader("X-FTL-CSRF", "Ux87YTIiMOf/GKCefVIOMw="); + xhr.send(data); + ``` + + === "Javascript (jQuery)" + + ``` javascript + $.ajax({ + url: "https://pi.hole/api/dns/blocking", + type: "GET", + data: null, + dataType: "json", + contentType: "application/json", + headers: { + "X-FTL-SID": "vFA+EP4MQ5JJvJg+3Q2Jnw=", + "X-FTL-CSRF": "Ux87YTIiMOf/GKCefVIOMw=" + } + }).done(function(data) { + console.log(data); + }).fail(function(xhr, status, error) { + console.log(error); + }); + ``` + + **Parameters** + + Specify the SID in one of the four ways described above. + + **Headers** + + If you use cookie-based authentication, you will also need to send a `X-FTL-CSRF` header with the CSRF token that was returned when you authenticated. + +## Authentication with 2FA + +If you have 2FA enabled for your Pi-hole, you will need to provide a TOTP token in addition to your password. You can generate this token using any TOTP app, e.g., [Google Authenticator](https://play.google.com/store/apps/details?id=com.google.android.apps.authenticator2) or [Authy](https://authy.com/download/). The Pi-hole API will return a `totp` flag in the response to indicate whether 2FA is enabled on your Pi-hole. + +???+ example "Authentication with 2FA" + + === "bash / cURL" + + ``` bash + curl -k -X POST "https://pi.hole/api/auth" --data '{"password":"your-password", "totp":"123456"}' + ``` + + === "Python 3" + + ``` python + import requests + + url = "https://pi.hole/api/auth" + payload = { + "password": "your-password", + "totp": 123456 + } + + response = requests.request("POST", url, json=payload, verify=False) + + print(response.text) + ``` + + === "Javascript (plain)" + + ``` javascript + var data = JSON.stringify({"password":"your-password", "totp":"123456"}); + var xhr = new XMLHttpRequest(); + + xhr.addEventListener("readystatechange", function () { + if (this.readyState === this.DONE) { + console.log(this.responseText); + } + }); + ``` + + === "Javascript (jQuery)" + + ``` javascript + $.ajax({ + url: "https://pi.hole/api/auth", + type: "POST", + data: JSON.stringify({"password":"your-password", "totp":"123456"}), + dataType: "json", + contentType: "application/json" + }).done(function(data) { + console.log(data); + }).fail(function(xhr, status, error) { + console.log(error); + }); + ``` + + **Parameters** + + Specify your password as `password` and the TOTP token you got from your TOTP app as `totp` in the payload. + +## Handling Authentication Errors + +When authenticating, it's possible to encounter errors. These can occur due to various reasons such as incorrect password, server issues, or network problems. Here's how you can handle these errors: + +When you send a `POST` request to the `/api/auth` endpoint, the server will respond with a status code. If the authentication is successful, you will receive a `200 OK` status code. If there's an error, you will receive a different status code. Here are some common ones: + +- `400 Bad Request`: This usually means that the JSON data in the request is not formatted correctly or the required fields are missing. +- `401 Unauthorized`: This means that the password provided is incorrect. +- `500 Internal Server Error`: This indicates that something went wrong on the server side (e.g., a system running out of memory). + +In addition to the status code, the server will also return a JSON object with more information about the error, e.g., + +???+ failure "Error response" + + Response code: `HTTP/1.1 400 - Bad Request` + + ```json + { + "error": { + "key": "bad_request", + "message": "No password found in JSON payload", + "hint": null + }, + "took": 0.0001 + } + ``` + + or + + ``` json + { + "error": { + "key": "bad_request", + "message": "Field password has to be of type 'string'", + "hint": null + }, + "took": 0.0003 + } + ``` + +## Session Expiration + +The session ID (SID) has a limited lifespan. Each successful request to the API will extend the lifespan of the SID. However, if there is a prolonged period of inactivity, the SID will expire and you will need to re-authenticate to obtain a new one. The timeout until the SID expires is configurable via a config setting. + +## Logging Out + +To end your session before the SID expires, you can send a `DELETE` request to the `/api/auth` endpoint. This will invalidate your current SID, requiring you to login again for further requests. + +???+ example "Logout" + + === "bash / cURL" + + ``` bash + # Example: Logout with SID in the request URI + curl -k -X DELETE "https://pi.hole/api/auth?sid=vFA+EP4MQ5JJvJg+3Q2Jnw=" + ``` + + === "Python 3" + + ``` python + # Example: Logout with SID in the request header + import requests + + url = "https://pi.hole/api/auth" + payload = {} + headers = { + "X-FTL-SID": "vFA+EP4MQ5JJvJg+3Q2Jnw=" + } + + response = requests.request("DELETE", url, headers=headers, data=payload, verify=False) + + print(response.text) + ``` + + === "Javascript (plain)" + + ``` javascript + var data = null; + var xhr = new XMLHttpRequest(); + + xhr.addEventListener("readystatechange", function () { + if (this.readyState === this.DONE) { + console.log(this.responseText); + } + }); + + xhr.open("DELETE", "https://pi.hole/api/auth?sid=vFA+EP4MQ5JJvJg+3Q2Jnw="); + xhr.send(data); + ``` + + === "Javascript (jQuery)" + + ``` javascript + $.ajax({ + url: "https://pi.hole/api/auth", + type: "DELETE", + data: null, + dataType: "json", + contentType: "application/json", + headers: { + "X-FTL-SID": "vFA+EP4MQ5JJvJg+3Q2Jnw=" + } + }).done(function(data) { + console.log(data); + }).fail(function(xhr, status, error) { + console.log(error); + }); + ``` + + **Parameters** + + Specify the SID in one of the four ways described above. + +???+ success "Success response" + + Response code: `HTTP/1.1 410 - Gone` + + No content + +Remember, it's important to manage your sessions properly to maintain the security of your Pi-hole API. + +## Rate Limiting + +The Pi-hole API implements a rate-limiting mechanism to prevent abuse. This means that you can try a certain number of login attempts per second. If you exceed this limit, you will receive a `429 Too Many Requests` response. + +## Limited number of concurrent sessions + +The Pi-hole API only allows a limited number of concurrent sessions. This means that if you try to login with a new session while the maximum number of sessions is already active, you may be denied access. This is to prevent abuse and resource exhaustion. In case you hit this limit, please make sure to logout from your sessions when you don't need them anymore as this will free up API slots for future requests. Unused sessions will be automatically terminated after a certain amount of time. + +## Security Implications of Session-Based Authentication + +Session-based authentication, while convenient and widely used, does have several security implications that Pi-hole addresses in the following ways: + +1. **Session Hijacking**: If an attacker manages to steal a user's session ID, they can impersonate that user for the duration of the session. This is mitigated indirectly by the methods #2-#4 below and also by binding sessions to the client's IP address. Be aware that this may cause issues if your IP address changes during the session's lifetime (e.g., a device reconnecting to a different WiFi network). + +2. **Cross-Site Scripting (`XSS`)**: XSS attacks can be used to steal session IDs if they are stored in JavaScript-accessible cookies. Pi-hole's session cookie is not accessible to JavaScript (`HttpOnly`), so this is not a concern. + +3. **Cross-Site Request Forgery (`CSRF`)**: In a CSRF attack, an attacker tricks a victim into performing actions on their behalf. This is mitigated by using CSRF tokens or implementing the `SameSite` attribute for the session cookie. + +4. **Session Fixation**: In this attack, an attacker provides a victim with a session ID, and if the victim authenticates with that session ID, the attacker can use it to impersonate the victim. This is mitigated by ever reusing session IDs but always regenerating them for each new session. Pi-hole sources the session ID from a cryptographically secure random number generator to ensure that it is unique, unpredictable, and safe. + +5. **Idle Sessions**: If a session remains open for a long time without activity, it can be hijacked by an attacker. This is mitigated by both the high security of the generated SIDs, making them basically non-bruteforceable, and by the fact that the session ID is bound to the client's IP address. This means that an attacker would not only have to be on the same network as the victim but even be on the same machine to hijack their session. This is furthermore mitigated by implementing session timeouts. + +Remember, no security measure is foolproof, but by understanding the potential risks and the multiple layers of defense your Pi-hole implemented against these risks, you can make an informed decision about how to use the Pi-hole API securely in the context of your own scripts. Always use the secure transmission method (HTTPS) offered by your Pi-hole to access the API. The strong encryption will prevent attackers from eavesdropping on your requests and makes stealing your session ID basically impossible. + +{!abbreviations.md!} diff --git a/docs/api/index.md b/docs/api/index.md new file mode 100644 index 000000000..5e2f92491 --- /dev/null +++ b/docs/api/index.md @@ -0,0 +1,203 @@ +The Pi-hole API is organized around [REST](http://en.wikipedia.org/wiki/Representational_State_Transfer). Our API has predictable resource-oriented URLs, accepts and returns reliable UTF-8 [JavaScript Object Notation (JSON)-encoded](http://www.json.org/) data for all API responses, and uses standard HTTP response codes and verbs. + +Most (but not all) endpoints require authentication. API endpoints requiring authentication will fail with code `401 Unauthorized` if no key is supplied. See [API Reference: Authentication](auth.md) for details. + +## Accessing the API documentation + +The entire API is documented at http://pi.hole/api/docs and self-hosted by your Pi-hole to match 100% the API versions your local Pi-hole has. Using this locally served API documentation is preferred. In case you don't have Pi-hole installed yet, you can also check out the documentation for all branches online, e.g., [Pi-hole API documentation](https://ftl.pi-hole.net/development-v6/docs/) (branch `development-v6`). Similarly, you can check out the documentation for a specific other branches by replacing `development-v6` with the corresponding branch name. + +## API endpoints + +An overview of all available endpoints is available at the API documentation page. The endpoints are grouped by their functionality. + +## JSON response + +The form of replies to successful requests strongly depends on the selected endpoint, e.g., + + +???+ success "Example reply: Success" + + Resource: `GET /api/dns/blocking` + + Response code: `HTTP/1.1 200 OK` + + ``` json + { + "blocking": true + } + ``` + + **Reply type** + + Object or Array + + **Fields** + + Depending on the particular endpoint + + +In contrast, errors have a uniform, predictable style to ease their programmatic treatment: + + +???+ failure "Example reply: Error (unauthorized access)" + + Resource: `GET /api/domains` + + Response code: `HTTP/1.1 401 Unauthorized` + + ``` json + { + "error": { + "key": "unauthorized", + "message": "Unauthorized", + "hint": null + } + } + ``` + + **Reply type** + + Object + + **Fields** + + ??? info "Key describing the error (`"key": string`)" + This string may be used for internal categorization of error types + + Examples for `key` are: + + - `bad_request` + + Possible reason: Payload is invalid for this endpoint + + - `database_error` + + Possible reason: Failed to read/write to the database + + ??? info "Human-readable description of the error (`"message": string`)" + This string may be shown to the user for troubleshooting + + Examples for `messages` are: + + - `Could not read domains from database table` + + Possible reason: Database is not readable + + - `No request body data` + + Possible reason: Payload is empty + + - `Invalid request body data` + + Possible reason: Payload is not valid JSON + + - `No "domain" string in body data` + + Possible reason: The required field `domain` is missing in the payload + + ??? info "Additional data (`"hint": [object|string|null]`)" + + The field `hint` may contain a JSON object or string. Its content depends on the error itself and may contain further details such as the interpreted user data. If no additional hint is available for this endpoint, `null` is returned. + + Examples for a failed request with `hint` being set is (domain is already on this list): + + ``` json + { + "error": { + "key": "database_error", + "message": "Could not add to gravity database", + "hint": { + "argument": "abc.com", + "enabled": true, + "sql_msg": "UNIQUE constraint failed: domainlist.domain, domainlist.type" + } + } + } + ``` + + +## HTTP methods used by this API + +Each HTTP request consists of a method that indicates the action to be performed on the identified resource. The relevant standards is [RFC 2616](https://tools.ietf.org/html/rfc2616). Though, RFC 2616 has been very clear in differentiating between the methods, complex wordings are a source of confusion for many users. + +Pi-hole's API uses the methods like this: + +Method | Description +---------|------------ +`GET` | **Read** from resource +`POST` | **Create** a resource +`PATCH` | **Update** existing resource +`PUT` | **Create or Replace** a resource +`DELETE` | **Delete** existing resource + + +??? info "Summarized details from [RFC 2616, Scn. 9](https://tools.ietf.org/html/rfc2616#section-9) (`GET/POST/PUT/DELETE`) and [RFC 2068, Scn. 19.6.1.1](https://datatracker.ietf.org/doc/html/rfc2068#section-19.6.1.1) (`PATCH`)" + ### `GET` + + The `GET` method means retrieve whatever information (in the form of an entity) that is identified by the URI. + + As `GET` requests do not change the state of the resource, these are said to be **safe methods**. Additionally, `GET` requests are **idempotent**, which means that making multiple identical requests must produce the same result every time until another method (`POST` or `PUT`) has changed the state of the resource on the server. + + For any given HTTP `GET`, if the resource is found on the server, then the API returns HTTP response code `200 (OK)` – along with the response body. + + In case a resource is NOT found on server, then the API returns HTTP response code `404 (Not found)`. Similarly, if it is determined that `GET` request itself is not correctly formed then API will return HTTP response code `400 (Bad request)`. + + ### `POST` + + Use `POST` APIs to **create new subordinate records**, e.g., a file is subordinate to a directory containing it or a row is subordinate to a database table. W`POST` methods are used to create a new resource into the collection of resources. + + If a resource has been created on the origin server, the response will be `201 (Created)`. + Not all action performed using the `POST` method will result in a resource that can be identified by a URI. In such a case, either `200 (OK)` or `204 (No Content)` is the appropriate response status, depending on whether or not the response includes an entity that describes the result. + + Note that `POST` is **neither safe nor idempotent**, and invoking two identical `POST` requests typically results in an error. + + ### `PUT` + + Use `PUT` primarily to **update existing records** (if the resource does not exist, the API will typically create a new record for it). If a new record has been added at the given URI, or an existing resource is modified, either the `200 (OK)` or `204 (No Content)` response codes are sent to indicate successful completion of the request. + + ### `PATCH` + + The `PATCH` method is similar to `PUT` except that the entity contains a list of differences between the original version of the resource identified by the Request-URI and the desired content of the resource after the `PATCH`` action has been applied. The list of differences is in a format defined by the media type of the entity (e.g., "application/diff") and MUST include sufficient information to allow the server to recreate the changes necessary to convert the original version of the resource to the desired version. + + ### `DELETE` + + As the name applies, `DELETE` APIs are used to **delete records** (identified by the Request-URI). + + A successful response will be `200 (OK)` if the response includes an entity describing the status, `202 (Accepted)` if the action has not yet been enacted, or `204 (No Content)` if the action has been enacted but the response does not include an entity. + + `DELETE` operations are **idempotent**. If you `DELETE` a resource, it’s removed from the collection of resources. Repeatedly calling `DELETE` on that resource will not change the outcome – however, calling `DELETE` on a resource a second time *may* return a 404 (NOT FOUND) since it was already removed. + +???+ info "Example" + Let’s list down few URIs and their purpose to get better understanding when to use which method: + + Method + URI | Interpretation + ---------------------|-------------------- + `GET /api/groups` | Get all groups + `POST /api/groups` | Create a new group + `GET /api/groups/abc` | Get the group `abc` + `PUT /api/groups/abc` | Update the group `abc` + `DELETE /api/groups/abc` | Delete group `abc` + + +## Error handling + +Pi-hole uses conventional HTTP response codes to indicate the success or failure of an API request. In general: Codes in the `2xx` range indicate success. Codes in the `4xx` range indicate an error that failed given the information provided (e.g., a required parameter was omitted, missing authentication, etc.). Codes in the `5xx` range indicate an error with Pi-hole's API (these are rare). + +Some `4xx` errors that could be handled programmatically include an error code that briefly explains the error reported: + +Code | Description | Interpretation +---- | ----------- | -------------- +`200` | `OK` | Everything worked as expected +`201` | `Content Created` | Added a new item +`204` | `No Content` | Removed an item +`400` | `Bad Request` | The request was unacceptable, often due to a missing required parameter +`401` | `Unauthorized` | No session identity provided for endpoint requiring authorization +`402` | `Request Failed` | The parameters were valid but the request failed +`403` | `Forbidden` | The API key doesn't have permissions to perform the request +`404` | `Not Found` | The requested resource doesn't exist +`429` | `Too Many Requests` | Too many requests hit the API too quickly +`500`, `502`, `503`, `504` | `Server Errors` | Something went wrong on Pi-hole's end (These are rare) + +We recommend writing code that gracefully handles all possible API exceptions. The Pi-hole API is designed to support this by standardized error messages and human-readable hints for errors. + +{!abbreviations.md!} diff --git a/docs/api/tls.md b/docs/api/tls.md new file mode 100644 index 000000000..6aa0f3a32 --- /dev/null +++ b/docs/api/tls.md @@ -0,0 +1,110 @@ +Pi-hole creates a self-signed certificate during installation. This certificate is used to encrypt the web interface and the API. While this certificate is secure, it is not trusted by your browser. This means that you will get a warning when you open the web interface or use the API like: + +![Warning in Firefox](../images/api/firefox-tls-insecure.png) + +You can avoid this warning by either using your own secure certificate and domain or by adding the certificate to your trusted certificates. + +## Adding the CA to your browser + +If you want to use the self-signed certificate, you can add the automatically generated certificate authority (CA) to your browser. This will make your browser trust all certificates signed by this CA. This is the same mechanism that is used by your browser to trust certificates signed by a certificate authority like Let's Encrypt. The following instructions show how to add the CA to Firefox and Chrome. The instructions for other browsers are similar. + +It is also possible to add the CA to your operating system's certificate store. This will make all applications that use the operating system's certificate store trust the CA. The instructions for this are operating system specific and are not covered here. + +Note that you have to add the **CA** certificate (e.g., `/etc/pihole/tls_ca.crt`) and not the server certificate (e.g., `/etc/pihole/tls.pem`). + +It is worth noting that the certificate is only valid for the domain that you had configured during certificate creation. If you access the Pi-hole web interface using a different domain, you will get a warning. This is because the certificate does not match the domain. You can either add the certificate for the other domain as well or you can create a new certificate for the other domain. You can easily create a new certificate by removing the old certificate and restarting `pihole-FTL` (e.g., `sudo rm /etc/pihole/tls* && sudo service pihole-FTL restart`). This will create a new certificate for the domain configured in `/etc/pihole/pihole.toml` (setting `webserver.domain`). + + +!!! warning "Security warning" + If you add the CA to your browser, you will trust all certificates signed by this CA. By discarding the CA's private key after certificate creation, the risk of a compromised CA is minimized. However, this also means you need to create (and, in turn, add to your OS or browser) a new CA whenever you change the domain of your Pi-hole installation. + + If you want to revoke the CA certificate itself, you can simply delete it from either your operating system's trust store or the browsers you have added it to at any time. + + +### Firefox (tested with Firefox 121.0) + +Before | After +:-----:|:-----: +![Firefox Untrusted](../images/api/firefox-pihole-untrusted.png) | ![Firefox Trusted](../images/api/firefox-pihole-trusted.png) + +1. Open the settings page of Firefox at [about:preferences#privacy](about:preferences#privacy) +2. Search for "Certificates" + ![Certificates in Firefox](../images/api/firefox-certificates.png) +3. Click on "View Certificates" +4. Select the "Authorities" tab +5. Click on "Import" and select the **CA** certificate file (e.g., `/etc/pihole/tls_ca.crt`) +6. Check "Trust this CA to identify websites" + ![Trust certificate in Firefox](../images/api/firefox-ca-trust.png) +7. Click on "OK" +8. Verify that the certificate has been imported correctly + ![Certificate added in Firefox](../images/api/firefox-ca-added.png) +9. Verify that the Pi-hole web interface is now trusted (no warning, secure lock icon) + +If the last step did not work, make sure that you have generated the certificate correctly (you may have a mismatch between the configured domain and the domain used by Pi-hole during certificate creation). Also verify that you have imported the **CA** certificate (not the server certificate). + +### Chrome (tested with Chrome 120.0) + +Before | After +:-----:|:-----: +![Chrome Untrusted](../images/api/chrome-pihole-untrusted.png) | ![Chrome Trusted](../images/api/chrome-pihole-trusted.png) + +1. Open the settings page of Chrome at [chrome://settings/privacy](chrome://settings/privacy) +2. Navigate to "Manage certificates" in the "Security" submenu of "Privacy and security" or use the search bar + ![Certificates in Chrome](../images/api/chrome-certificates.png) +3. Click on "Authorities" tab +4. Click on "Import" and select the **CA** certificate file (e.g., `/etc/pihole/tls_ca.crt`) +5. Check "Trust this certificate for identifying websites" + ![Trust certificate in Chrome](../images/api/chrome-ca-trust.png) +6. Click on "OK" +7. Verify that the certificate has been imported correctly + ![Certificate added in Chrome](../images/api/chrome-ca-added.png) +8. Verify that the Pi-hole web interface is now trusted (no warning, secure lock icon) + +If the last step did not work, see the remark below the Firefox instructions above. + +### Android (tested with Android 11 and Firefox Mobile 121.1.0) + +Before | After +:-----:|:-----: +![Android Firefox Untrusted](../images/api/android-pihole-untrusted.png) | ![Android Firefox Trusted](../images/api/android-pihole-trusted.png) +![Android Chrome Untrusted](../images/api/android-chrome-untrusted.png) | ![Android Chrome Trusted](../images/api/android-chrome-trusted.png) + +1. Go to you device's settings +2. Navigate to "System Security" or "Security & location" (depending on your device) +3. Navigate to "Credential storage" or similar (depending on your device) + + ![Android System Security menu](../images/api/android-system-security.png) + +4. Choose "Install certificates from storage" or similar (depending on your device) + + ![Android Credential storage menu](../images/api/android-credential-storage.png) + +5. Select "User certificates" or similar (depending on your device) + + ![Android Install certificates menu](../images/api/android-install-ca.png) + +6. Select the **CA** certificate file (e.g., `/etc/pihole/tls_ca.crt`) - you need to have the certificate file on your device (e.g., by copying it to your device via Bluetooth or other *secure* means) + + ![Android Select certificate file](../images/api/android-ca-select.png) + +7. Confirm that you want to install the certificate despite the possible security implications mentioned above + + ![Android Confirm certificate installation](../images/api/android-ca-security-warning.png) + +8. Give the certificate a meaningful name (e.g., "Pi-hole") + + ![Android Name certificate](../images/api/android-ca-name.png) + +9. Verify that the certificate has been imported correctly by checking for a small success popup in the lower area of the screen + + ![Android Certificate imported](../images/api/android-ca-success.png) + +10. Verify that the Pi-hole web interface is now trusted (no warning, secure lock icon) + +The certificate will be valid for all apps that use the Android certificate store. This includes the Firefox Mobile browser but also others such as Chrome. + +If the last step did not work, see the remark below the Firefox instructions above. + +## Using your own certificate + +If you want to use your own certificate, you can do so by placing the certificate and the private key in a location that can be read by user `pihole` (e.g., `/etc/pihole`) and, change the path in `/etc/pihole/pihole.toml` (setting `webserver.api.tls.cert`) and restart `pihole-FTL` (e.g., `sudo service pihole-FTL restart`). The certificate and the private key must be in PEM format (check automatically generated certificate for an example). diff --git a/docs/core/chronometer-thumb.jpg b/docs/core/chronometer-thumb.jpg deleted file mode 100644 index 95171f70a..000000000 Binary files a/docs/core/chronometer-thumb.jpg and /dev/null differ diff --git a/docs/core/chronometer-thumb@2x.jpg b/docs/core/chronometer-thumb@2x.jpg deleted file mode 100644 index 7a17ce952..000000000 Binary files a/docs/core/chronometer-thumb@2x.jpg and /dev/null differ diff --git a/docs/core/chronometer.jpg b/docs/core/chronometer.jpg deleted file mode 100644 index 45bca264b..000000000 Binary files a/docs/core/chronometer.jpg and /dev/null differ diff --git a/docs/database/gravity/groups.md b/docs/database/domain-database/groups.md similarity index 95% rename from docs/database/gravity/groups.md rename to docs/database/domain-database/groups.md index 8c9dc33cc..d81428d04 100644 --- a/docs/database/gravity/groups.md +++ b/docs/database/domain-database/groups.md @@ -1,4 +1,6 @@ -## Group management +--- +title: Group Management +--- Groups are defined in the `group` table and can have an optional description in addition to the mandatory name of the group. @@ -12,7 +14,7 @@ Label | Type | Uniqueness enforced | Content Group management is implemented using so-called linking tables. Hence, it is possible to - associate domains (and clients!) with any number of groups, -- manage adlists together with groups, +- manage lists together with groups, - use the same groups for black- and whitelisted domains at the same time. The linking tables are particularly simple, as they only link group `id`s with list `id`s. As an example, we describe the `domainlist_by_group` table. The `adlist` and `client` linking tables are constructed similarly. diff --git a/docs/database/gravity/index.md b/docs/database/domain-database/index.md similarity index 74% rename from docs/database/gravity/index.md rename to docs/database/domain-database/index.md index 4f78291fd..f92c94da4 100644 --- a/docs/database/gravity/index.md +++ b/docs/database/domain-database/index.md @@ -1,8 +1,23 @@ -Pi-hole uses the well-known relational database management system SQLite3 for managing the various domains that are used to control the DNS filtering system. The database-based domain management has been added with Pi-hole v5.0. +--- +title: Domain Database +--- + +Pi-hole uses the well-known relational database management system SQLite3 for managing the various domains that are used to control the DNS filtering system. The database-based domain management has been added with Pi-hole v5.0. The ability to subscribe to external *allow*lists has been added with Pi-hole v6.0. + +## Priorities + +Pi-hole uses the following priorities when deciding whether to block or allow a domain: + +1. Exact allowlist entries +2. Regex allowlist entries +3. Exact blocklist entries +4. Subscribed allowlists +5. Subscribed blocklists +6. Regex blocklist entries ## Domain tables (`domainlist`) -The database stores white-, and blacklists which are directly relevant for Pi-hole's domain blocking behavior. The `domainlist` table contains all domains on the white- and blacklists. It has a few extra fields to store data related to a given domain such as the `enabled` state, the dates when the domain was added and when it was last modified, and an optional comment. +The database stores allow-, and blocklists which are directly relevant for Pi-hole's domain blocking behavior. The `domainlist` table contains all domains on the allow- and blocklists. It has a few extra fields to store data related to a given domain such as the `enabled` state, the dates when the domain was added and when it was last modified, and an optional comment. The date fields are defined as `INTEGER` fields as they expect numerical timestamps also known as *UNIX time*. The `date_added` and `date_modified` fields are initialized with the current timestamp converted to UNIX time. The `comment` field is optional and can be empty. @@ -11,14 +26,14 @@ Pi-hole's *FTL*DNS reads the tables through the various view, omitting any disab Label | Type | Uniqueness enforced | Content ----- | ---- | ------------------- | -------- `id` | integer | Yes | Unique ID for database operations -`type` | integer | No | `0` = exact whitelist,
`1` = exact blacklist,
`2` = regex whitelist,
`3` = regex blacklist +`type` | integer | No | `0` = exact allowlist,
`1` = exact blocklist,
`2` = regex allowlist,
`3` = regex blocklist `domain` | text | Yes | Domain `enabled` | boolean | No | Flag whether domain should be used by `pihole-FTL`
(`0` = disabled, `1` = enabled) `date_added` | integer | No | Timestamp when domain was added `date_modified` | integer | No | Timestamp when domain was last modified, automatically updated when a record is changed `comment` | text | No | Optional field for arbitrary user comments, only field that is allowed to be `NULL` -## Adlist Table (`adlist`) +## List Table (`adlist`) The `adlist` table contains all sources for domains to be collected by `pihole -g`. Just like the other tables, it has a few extra fields to store metadata related to a given source. @@ -35,18 +50,18 @@ Label | Type | Uniqueness enforced | Content `invalid_domains` | integer | No | Number of invalid domains on this list `status` | integer | No | `1` = List download was successful, `2` = List unchanged upstream, Pi-hole used a local copy, `3` = List unavailable, Pi-hole used a local copy, `4` = List unavailable, there is no local copy of this list available on your Pi-hole -## Gravity Table (`gravity`) +## Gravity Tables (`gravity` and `antigravity`) -The `gravity` table consists of the domains that have been processed by Pi-hole's `gravity` (`pihole -g`) command. The domains in this list are the collection of domains sourced from the configured sources (see the [`adlist` table](index.md#adlist-table-adlist). +The `gravity` and `antigravity` table consists of the domains that have been processed by Pi-hole's `gravity` (`pihole -g`) command. The domains in this list are the collection of domains sourced from the configured sources (see the [List Table (`adlist`)](index.md#list-table-adlist)). -During each run of `pihole -g`, this table is flushed and completely rebuilt from the newly obtained set of domains to be blocked. +During each run of `pihole -g`, these tables are flushed and completely rebuilt from the newly obtained set of domains to be blocked or allowed. Label | Type | Content ----- | ---- | ------- -`domain` | text | Blocked domain compiled from adlist referenced by `adlist_id` -`adlist_id` | integer | ID associated to adlists in table `adlist` +`domain` | text | Domain compiled from subscribed list referenced by `adlist_id` +`adlist_id` | integer | ID associated to subscribed list in table `adlist` -Uniqueness is enforced on pairs of (`domain`, `adlist_id`). In other words: domains can be added multiple times, however, only when they are referencing different adlists as their origins. +Uniqueness is enforced on pairs of (`domain`, `adlist_id`) in both tables. In other words: domains can be added multiple times, however, only when they are referencing different lists as their origins. ## Client table (`client`) @@ -70,13 +85,3 @@ This allows to specify "broad clients" such as CIDR notation can be used for IPv6 subnets as well. The IPv6 block `2001:db8::/48` represents all IPv6 addresses from `2001:db8:0:0:0:0:0:0` to `2001:db8:0:ffff:ffff:ffff:ffff:ffff` (1,208,925,819,614,629,174,706,176 = roughly one heptillion devices). Note that Pi-hole's implementation is more generic than what is written on the linked Wikipedia article as you can use *any* CIDR block (not only multiples of 4). - -## Audit Table (`domain_audit`) - -The `domain_audit` table contains domains that have been audited by the user on the web interface. - -Label | Type | Uniqueness enforced | Content ------ | ---- | ------------------- | -------- -`id` | integer | Yes | Unique ID for database operations -`domain` | text | Yes | Domain -`date_added` | integer | No | Unix timestamp when domain was added diff --git a/docs/database/gravity/recovery.md b/docs/database/domain-database/recovery.md similarity index 100% rename from docs/database/gravity/recovery.md rename to docs/database/domain-database/recovery.md diff --git a/docs/database/index.md b/docs/database/index.md index dea18ea45..b9c630db8 100644 --- a/docs/database/index.md +++ b/docs/database/index.md @@ -2,5 +2,5 @@ Pi-hole uses the well-known relational database management system SQLite3 both f Details concerning the databases, their contained tables and exemplary SQL commands allowing even complex requests to Pi-hole's databases are described on the subpages of this category. -- [Query database `/etc/pihole/pihole-FTL.db`](ftl.md) -- [Domain database `/etc/pihole/gravity.db`](gravity/index.md) +- [Query database `/etc/pihole/pihole-FTL.db`](query-database.md) +- [Domain database `/etc/pihole/gravity.db`](domain-database/index.md) diff --git a/docs/database/ftl.md b/docs/database/query-database.md similarity index 87% rename from docs/database/ftl.md rename to docs/database/query-database.md index 48719c94e..aae8bbc8a 100644 --- a/docs/database/ftl.md +++ b/docs/database/query-database.md @@ -44,15 +44,16 @@ Label | Type | Allowed to by empty | Content --- | --- | ---- | ----- `id` | integer | No | autoincrement ID for the table, only used by SQLite3, not by *FTL*DNS `timestamp` | integer | No | Unix timestamp when this query arrived at *FTL*DNS (used as index) -`type` | integer | No | Type of this query (see [Supported query types](ftl.md#supported-query-types)) -`status` | integer | No | How was this query handled by *FTL*DNS? (see [Supported status types](ftl.md#supported-status-types)) +`type` | integer | No | Type of this query (see [Supported query types](query-database.md#supported-query-types)) +`status` | integer | No | How was this query handled by *FTL*DNS? (see [Supported status types](query-database.md#supported-status-types)) `domain` | text | No | Requested domain `client` | text | No | Requesting client (IP address) `forward` | text | Yes | Forward destination used for this query (only set if `status == 2`) `additional_info` | blob | Yes | Data-dependent content, see below -`reply_type` | integer | Yes | Type of the reply for this query (see [Supported reply types](ftl.md#supported-reply-types)) +`reply_type` | integer | Yes | Type of the reply for this query (see [Supported reply types](query-database.md#supported-reply-types)) `reply_time` | real | Yes | Seconds it took until the final reply was received -`dnssec` | integer | Yes | Type of the DNSSEC status for this query (see [DNSSEC status](ftl.md#dnssec-status)) +`dnssec` | integer | Yes | Type of the DNSSEC status for this query (see [DNSSEC status](query-database.md#dnssec-status)) +`regex_id` | integer | Yes | ID of the regex filter that matched this query (only set if blocked by a regex filter) The `queries` `VIEW` is dynamically generated from the data actually stored in the `query_storage` table and the linking tables `domain_by_id`, `client_by_id`, `forward_by_id`, and `addinfo_by_id` (see below). The table `query_storage` will contains integer IDs pointing to the respective entries of the linking tables to save space and make searching the database faster. If you haven't upgraded for some time, the table may still contain strings instead of integer IDs. @@ -66,7 +67,7 @@ If a query was blocked due to a CNAME inspection (status 9, 10, 11), this field ##### Query influenced by a black- or whitelist entry {#additional_info_list data-toc-label='domainlist_id'} -If a query was influenced by a black- or whitelist entry, this field contains the ID of the corresponding entry in the [`domainlist`](gravity/index.md#domain-tables-domainlist) table. +If a query was influenced by a black- or whitelist entry, this field contains the ID of the corresponding entry in the [`domainlist`](domain-database/index.md#domain-tables-domainlist) table. ### Counters table @@ -114,23 +115,24 @@ Any other query type will be stored with an offset of 100, i.e., `TYPE66` will b ID | Status | | Details --- | --- | --- | --- 0 | Unknown | ❔ | Unknown status (not yet known) -1 | Blocked | ❌ | Domain contained in [gravity database](gravity/index.md#gravity-table-gravity) +1 | Blocked | ❌ | Domain contained in [gravity database](domain-database/index.md#gravity-tables-gravity-and-antigravity) 2 | Allowed | ✅ | Forwarded 3 | Allowed | ✅ | Replied from cache -4 | Blocked | ❌ | Domain matched by a [regex blacklist](gravity/index.md#domain-tables-domainlist) filter -5 | Blocked | ❌ | Domain contained in [exact blacklist](gravity/index.md#domain-tables-domainlist) +4 | Blocked | ❌ | Domain matched by a [regex blacklist](domain-database/index.md#domain-tables-domainlist) filter +5 | Blocked | ❌ | Domain contained in [exact blacklist](domain-database/index.md#domain-tables-domainlist) 6 | Blocked | ❌ | By upstream server (known blocking page IP address) 7 | Blocked | ❌ | By upstream server (`0.0.0.0` or `::`) 8 | Blocked | ❌ | By upstream server (`NXDOMAIN` with `RA` bit unset) -9 | Blocked | ❌ | Domain contained in [gravity database](gravity/index.md#gravity-table-gravity)
*Blocked during deep CNAME inspection* -10 | Blocked | ❌ | Domain matched by a [regex blacklist](gravity/index.md#domain-tables-domainlist) filter
*Blocked during deep CNAME inspection* -11 | Blocked | ❌ | Domain contained in [exact blacklist](gravity/index.md#domain-tables-domainlist)
*Blocked during deep CNAME inspection* +9 | Blocked | ❌ | Domain contained in [gravity database](domain-database/index.md#gravity-tables-gravity-and-antigravity)
*Blocked during deep CNAME inspection* +10 | Blocked | ❌ | Domain matched by a [regex blacklist](domain-database/index.md#domain-tables-domainlist) filter
*Blocked during deep CNAME inspection* +11 | Blocked | ❌ | Domain contained in [exact blacklist](domain-database/index.md#domain-tables-domainlist)
*Blocked during deep CNAME inspection* 12 | Allowed | ✅ | Retried query 13 | Allowed | ✅ | Retried but ignored query (this may happen during ongoing DNSSEC validation) 14 | Allowed | ✅ | Already forwarded, not forwarding again 15 | Blocked | ❌ | Blocked (database is busy)
How these queries are handled can be [configured](../ftldns/configfile.md#reply_when_busy) 16 | Blocked | ❌ | Blocked (special domain)
*E.g. Mozilla's canary domain and Apple's Private Relay domains*
Handling can be [configured](../ftldns/configfile.md) 17 | Allowed | ✅⌛ | Replied from *stale* cache +18 | Blocked | ❌ | By upstream server (EDE 15) ### Supported reply types @@ -160,6 +162,7 @@ ID | DNSSEC status is 2 | `INSECURE` 3 | `BOGUS` 4 | `ABANDONED` +5 | `TRUNCATED` ### Linking tables @@ -198,7 +201,7 @@ Label | Type | Allowed to by empty | Content Valid `type` IDs are currently - `ADDINFO_CNAME_DOMAIN = 1` - `content` is a string (the related CNAME) -- `ADDINFO_DOMAIN_ID = 2` - `content` is an integer (ID pointing to a domain in the [`domainlist` table](gravity/index.md#domain-tables-domainlist)) +- `ADDINFO_DOMAIN_ID = 2` - `content` is an integer (ID pointing to a domain in the [`domainlist` table](domain-database/index.md#domain-tables-domainlist)) ### Example for interaction with the long-term query database diff --git a/docs/docker/build-image.md b/docs/docker/build-image.md new file mode 100644 index 000000000..7bff5cf14 --- /dev/null +++ b/docs/docker/build-image.md @@ -0,0 +1,79 @@ +In case you wish to customize the image, or perhaps check out a branch after being asked by a developer to do so, you can use the convenient `build.sh` script located in the root of the [docker-pi-hole repository](https://github.com/pi-hole/docker-pi-hole) + +## Checking out the repository + +In order to build the image locally, you will first need a copy of the repository on your computer. The following commands will clone the repository from Github and then put you into the directory + +``` +git clone https://github.com/pi-hole/docker-pi-hole +cd docker-pi-hole +git checkout development #NOTE: This step is only needed until V6 is released +``` + +All other commands following assume you have at least run the above steps. + +## Build.sh + +``` +Usage: ./build.sh [-l] [-f ] [-c ] [-w ] [-t ] [use_cache] +Options: + -f, --ftlbranch Specify FTL branch (cannot be used in conjunction with -l) + -c, --corebranch Specify Core branch + -w, --webbranch Specify Web branch + -p, --paddbranch Specify PADD branch + -t, --tag Specify Docker image tag (default: pihole:local) + -l, --local Use locally built FTL binary (requires src/pihole-FTL file) + use_cache Enable caching (by default --no-cache is used) + +If no options are specified, the following command will be executed: + docker buildx build src/. --tag pihole:local --load --no-cache +``` + +## Example uses of the script + +### Contributing to the development of `docker-pi-hole` + +When contributing, it's always a good idea to test your changes before submitting a pull request. Simply running `./build.sh` will allow you to do so. + +There is also `./build-and-test.sh`, which can be used to verify the tests that are run on Github pass with your changes. + +``` +git checkout -b myNewFeatureBranch +#make some changes +./build.sh +``` + +### As an alternative to `pihole checkout` + +Occasionally you may need to try an alternative branch of one of the components (`core`,`web`,`ftl`). On bare metal you would run, for example, `pihole checkout core branchName`, however in the Docker image we have disabled this command as it can cause unpredictable results. + +#### Examples + +- You have been asked by a developer to checkout the FTL branch `new/Sensors`. To do so + +``` +./build.sh -f new/Sensors +``` + +- There is new docker-specific work being carried out on the branch `fix/logRotate` that you wish to test + +``` +git checkout fix/logRotate +./build.sh +``` + +## Using the built image + +Unless otherwise named via the `-t` command, the script will build an image locally and tag it as `pihole:local`. You can reference this as a drop-in replacement for `pihole/pihole:latest` in your compose file or your run command: + +```yaml +services: + pihole: + container_name: pihole + image: pihole:local +... +``` + +``` +docker run [options] pihole:local +``` diff --git a/docs/docker/configuration.md b/docs/docker/configuration.md new file mode 100644 index 000000000..e4e03412b --- /dev/null +++ b/docs/docker/configuration.md @@ -0,0 +1,85 @@ +# Configuration + +The recommended way to configure the Pi-hole docker container is by utilizing [environment variables](https://docs.docker.com/compose/how-tos/environment-variables/), however if you are persisting your `/etc/pihole` directory, you choose instead to set them via the web interface, or by directly editing `pihole.toml` + +## Environment Variables + +### Recommended Variables + +#### `TZ` (Default: `UTC`) + +Set your [timezone](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones) to make sure logs rotate at local midnight instead of at UTC midnight. + +#### `FTLCONF_webserver_api_password` (Default: `unset`) + +To set a specific password for the web interface, use the environment variable `FTLCONF_webserver_api_password` (per the quick-start example). If this variable is not detected, and you have not already set one previously inside the container via `pihole setpassword` or `pihole-FTL --config webserver.api.password`, then a random password will be assigned on startup, and will be printed to the log. You can find this password with the command `docker logs pihole | grep random password` on your host to find this password. + +#### `FTLCONF_dns_upstreams` (Default: `8.8.8.8;8.8.4.4`) + +- Upstream DNS server(s) for Pi-hole to forward queries to, separated by a semicolon +- Supports non-standard ports with #[port number] e.g `127.0.0.1#5053;8.8.8.8;8.8.4.4` +- Supports Docker service names and links instead of IPs e.g `upstream0;upstream1` where upstream0 and upstream1 are the service names of or links to docker services + +### Configuring FTL Via The Environment + +While FTL's configuration file can be manually edited, set via the CLI (`pihole-FTL --config setting.name=value`), or set via the web interface - the recommended approach is to do this via environment variables + +As with the recommended examples above for the web password and DNS upstreams, the syntax is `FTLCONF_[section_][setting]` + +Given the below `toml` formatted example from `pihole.toml`, we can translate this to the environment variable `FTLCONF_dns_dnssec` + +```toml +[dns] + dnssec = true +``` + +!!! note + All FTL settings that are set via environment variables effectively become read-only, meaning that you will not be able to change them via the web interface or CLI. This is to ensure a "single source of truth" on the config. If you later unset or remove an environment variable, then FTL will revert to the default value for that setting + + +An example of how some of these variables may look in your compose file + +```yaml + environment: + TZ: europe/London + FTLCONF_dns_revServers: 'true,192.168.0.0/16,192.168.0.1,lan' + FTLCONF_dns_upstreams: '8.8.8.8;8.8.4.4' + FTLCONF_webserver_api_password: 'correct horse battery staple' + FTLCONF_webserver_port: '8082,443s' + FTLCONF_debug_api: 'true' +``` + +### Other Variables + +#### `TAIL_FTL_LOG` (Default: `1`) + +Whether or not to output the FTL log when running the container. Can be disabled by setting the value to 0 + +#### `PIHOLE_UID` (Default: `100`) + +Overrides image's default pihole user id to match a host user id + +#### `PIHOLE_GID` (Default: `100`) + +Overrides image's default pihole group id to match a host group id + +!!! Warning + For the above two settings, the `id` must not already be in use inside the container! + +#### `FTL_CMD` (Default: `no-daemon`) + +Customize the options with which dnsmasq gets started. e.g. `no-daemon -- --dns-forward-max 300` to increase max. number of concurrent dns queries on high load setups. + +#### `DNSMASQ_USER` (Default: `pihole`) + +Allows changing the user that FTLDNS runs as. Default: pihole, some systems such as Synology NAS may require you to change this to root (See [pihole/docker-pi-hole#963](https://github.com/pi-hole/docker-pi-hole/issues/963)) + +#### `ADDITIONAL_PACKAGES` (Default: unset) + +Mostly for development purposes, this just makes it easier for those of us that always like to have whatever additional tools we need inside the container for debugging. + +Adding packages here is the same as running `apk add ` inside the container + +#### `PH_VERBOSE` (Default: `0`) + +Setting this environment variable to `1` will set `-x`, making the scripts that run on container startup more verbose. Useful for debugging only. diff --git a/docs/docker/index.md b/docs/docker/index.md new file mode 100644 index 000000000..4851b8a7f --- /dev/null +++ b/docs/docker/index.md @@ -0,0 +1,66 @@ +The easiest way to get up and running with Pi-hole on Docker is to use our quick-start `docker-compose.yml` template. + +Copy the below [Docker Compose](https://docs.docker.com/compose/) example and customize as needed + +```yaml +# More info at https://github.com/pi-hole/docker-pi-hole/ and https://docs.pi-hole.net/ +services: + pihole: + container_name: pihole + image: pihole/pihole:latest + ports: + # DNS Ports + - "53:53/tcp" + - "53:53/udp" + # Default HTTP Port + - "80:80/tcp" + # Default HTTPs Port. FTL will generate a self-signed certificate + - "443:443/tcp" + # Uncomment the below if using Pi-hole as your DHCP Server + #- "67:67/udp" + environment: + # Set the appropriate timezone for your location from + # https://en.wikipedia.org/wiki/List_of_tz_database_time_zones, e.g: + TZ: 'Europe/London' + # Set a password to access the web interface. Not setting one will result in a random password being assigned + FTLCONF_webserver_api_password: 'correct horse battery staple' + # If using Docker's default `bridge` network setting the dns listening mode should be set to 'all'3 + FTLCONF_dns_listeningMode: 'all' + # Volumes store your data between container upgrades + volumes: + # For persisting Pi-hole's databases and common configuration file + - './etc-pihole:/etc/pihole' + # For persisting custom dnsmasq config files. Most will not need this, and can be safely removed/commented out + - './etc-dnsmasq.d:/etc/dnsmasq.d' + cap_add: + # Required if you are using Pi-hole as your DHCP server, else not needed + # See Note On Capabilities below + - NET_ADMIN + restart: unless-stopped +``` + +Run `docker compose up -d` to build and start Pi-hole (on older systems, the syntax here may be `docker-compose up -d`) + +The equivalent command for `docker run` would be: + +``` +docker run --name pihole -p 53:53/tcp -p 53:53/udp -p 80:80/tcp -p 443:443/tcp -e TZ=Europe/London -e FTLCONF_webserver_api_password="correct horse battery staple" -e FTLCONF_dns_listeningMode=all -v ./etc-pihole:/etc/pihole -v ./etc-dnsmasq.d:/etc/dnsmasq.d --cap-add NET_ADMIN --restart unless-stopped pihole/pihole:latest +``` + +## Note On Capabilities + +[FTLDNS](https://docs.pi-hole.net/ftldns/in-depth/#linux-capabilities) expects to have the following capabilities available: + +- `CAP_NET_BIND_SERVICE`: Allows FTLDNS binding to TCP/UDP sockets below 1024 (specifically DNS service on port 53) +- `CAP_NET_RAW`: use raw and packet sockets (needed for handling DHCPv6 requests, and verifying that an IP is not in use before leasing it) +- `CAP_NET_ADMIN`: modify routing tables and other network-related operations (in particular inserting an entry in the neighbor table to answer DHCP requests using unicast packets) +- `CAP_SYS_NICE`: FTL sets itself as an important process to get some more processing time if the latter is running low +- `CAP_CHOWN`: we need to be able to change ownership of log files and databases in case FTL is started as a different user than `pihole` +- `CAP_SYS_TIME`: FTL needs to be able to set the system time to update it using the Network Time Protocol (NTP) in the background + +!!! info + This image automatically grants those capabilities, if available, to the FTLDNS process, even when run as non-root. + +By default, docker does not include the `NET_ADMIN` capability for non-privileged containers, and it is recommended to explicitly add it to the container using `--cap-add=NET_ADMIN`. + +However, if DHCP and IPv6 Router Advertisements are not in use, it should be safe to skip it. For the most paranoid, it should even be possible to explicitly drop the `NET_RAW` capability to prevent FTLDNS from automatically gaining it. diff --git a/docs/docker/upgrading/index.md b/docs/docker/upgrading/index.md new file mode 100644 index 000000000..42b2280e4 --- /dev/null +++ b/docs/docker/upgrading/index.md @@ -0,0 +1,31 @@ +# Upgrading + +The standard Pi-hole customization abilities apply to this docker, but with docker twists such as using docker volume mounts to map host stored file configurations over the container defaults. However, mounting these configuration files as read-only should be avoided. Volumes are also important to persist the configuration in case you have removed the Pi-hole container which is a typical docker upgrade pattern. + +!!! tip "Always Read The Release Notes!" + Ensure you read the release notes for both the Docker Release and the main Pi-hole component releases. This will help you avoid common problems due to known issues with upgrading or newly required arguments or variables + +## Upgrading / Reconfiguring + +!!! Note + The normal Pi-hole functions to upgrade (`pihole -up`) or reconfigure (`pihole -r`) are disabled within the docker container. New images will be released, and you can upgrade by replacing your old container with a fresh upgraded image, which is more in line with the 'docker way'. Long-living docker containers are not the docker way since they aim to be portable and reproducible, why not re-create them often! Just to prove you can. + +### Docker Compose + +Navigate to the directory in which your `docker-compose.yml` file exists and run the following commands + +```bash +docker compose down +docker compose pull +docker compose up -d +``` + +### Docker run + +```bash +docker stop pihole +docker rm pihole +docker pull pihole/pihole:latest +docker run [ ... arguments (see Getting Started) ... ] +``` + diff --git a/docs/docker/upgrading/v5-v6.md b/docs/docker/upgrading/v5-v6.md new file mode 100644 index 000000000..d74c46c48 --- /dev/null +++ b/docs/docker/upgrading/v5-v6.md @@ -0,0 +1,85 @@ +!!! info + The last image of the Pi-hole docker container which contained v5.x Pi-hole components was [`2024.07.0`](https://github.com/pi-hole/docker-pi-hole/releases/tag/2024.07.0) +

+ Versions after this will be based on v6.x Pi-hole components. + +## Changed Environment Variables + +Nearly all environment variables from previous versions have been removed, but functionality can be replicated by setting their `FTLCONF_` equivalent instead. (See [Configuration](../configuration.md#configuring-ftl-via-the-environment) page for more details) + +### Misc + +| Old Variable | New Variable | +|:-------------|:-------------| +|`CUSTOM_CACHE_SIZE`|`FTLCONF_dns_cache_size`| +|`DNS_BOGUS_PRIV`|`FTLCONF_dns_bogusPriv`| +|`DNS_FQDN_REQUIRED`|`FTLCONF_dns_domainNeeded`| +|`DNSMASQ_LISTENING`|`FTLCONF_dns_listeningMode`| +|`DNSSEC`|`FTLCONF_dns_dnssec`| +|`INTERFACE`|`FTLCONF_dns_interface`| +|`PIHOLE_DNS_`|`FTLCONF_dns_upstreams`| +|`QUERY_LOGGING`|`FTLCONF_dns_queryLogging`| +|`WEBPASSWORD`|`FTLCONF_webserver_api_password`| +|`WEB_PORT`|`FTLCONF_webserver_port`| +|`WEBTHEME`|`FTLCONF_webserver_interface_theme`| +|`WEBUIBOXEDLAYOUT`|`FTLCONF_webserver_interface_boxed`| + +### DHCP Variables + +|Old|New| +|:--|:--| +|`DHCP_ACTIVE`| `FTLCONF_dhcp_active`| +|`DHCP_START`| `FTLCONF_dhcp_start`| +|`DHCP_END`| `FTLCONF_dhcp_end`| +|`DHCP_ROUTER`| `FTLCONF_dhcp_router`| +|`DHCP_LEASETIME`| `FTLCONF_dhcp_leaseTime`| +|`PIHOLE_DOMAIN`| `FTLCONF_dns_domain`| +|`DHCP_IPv6`| `FTLCONF_dhcp_ipv6`| +|`DHCP_rapid_commit`| `FTLCONF_dhcp_rapidCommit`| + +### Conditional Forwarding Variables + +Reverse server (`REV_SERVER*`) variables should be replaced with the single `FTLCONF_dns_revServers`, which accepts an array of reverse servers (delimited by `;`) in the following format: + +``` +,[/],[#], +``` + +If, for example, you had the following old style variables: + +- `REV_SERVER: 'true'` +- `REV_SERVER_DOMAIN: 'lan'` +- `REV_SERVER_TARGET: '192.168.0.1#53'` +- `REV_SERVER_CIDR: '192.168.0.0/24'` + +You would now set: + +``` +FTLCONF_dns_revServers: 'true,192.168.0.0/24,192.168.0.1#53,lan' +``` + +### Removed Variables + +The following variables can be removed without replacement, as they are no longer used: + +- `CONDITIONAL_FORWARDING` +- `CONDITIONAL_FORWARDING_DOMAIN` +- `CONDITIONAL_FORWARDING_IP` +- `CONDITIONAL_FORWARDING_REVERSE` +- `CORS_HOSTS` +- `DNS1` +- `DNS2` +- `FTLCONF_REPLY_ADDR4` +- `FTLCONF_REPLY_ADDR6` +- `IPV6` +- `ServerIP` +- `ServerIPv6` +- `SKIPGRAVITYONBOOT` +- `TEMPERATUREUNIT` +- `VIRTUAL_HOST` +- `WEB_BIND_ADDR` +- `WEB_GID` +- `WEB_UID` +- `WEBLOGS_STDOUT` +- `WEBPASSWORD_FILE` + diff --git a/docs/ftldns/compile.md b/docs/ftldns/compile.md index 317bdaa87..9b8a41956 100644 --- a/docs/ftldns/compile.md +++ b/docs/ftldns/compile.md @@ -12,13 +12,13 @@ Install them by running the following command in a terminal: ### Debian / Ubuntu / Raspbian ```bash -sudo apt install git wget ca-certificates build-essential libgmp-dev m4 cmake libidn11-dev libreadline-dev xxd +sudo apt install git wget ca-certificates build-essential libgmp-dev m4 cmake libidn2-dev libunistring-dev libreadline-dev xxd ``` ### Fedora ```bash -sudo dnf install git wget ca-certificates gcc gmp-devel gmp-static m4 cmake libidn-devel readline-devel xxd +sudo dnf install git wget ca-certificates gcc gmp-devel gmp-static m4 cmake libidn2-devel libunistring-devel readline-devel xxd ``` ## Compile `libnettle` from source @@ -27,16 +27,34 @@ sudo dnf install git wget ca-certificates gcc gmp-devel gmp-static m4 cmake libi Compile and install a recent version using: ```bash -wget https://ftp.gnu.org/gnu/nettle/nettle-3.8.1.tar.gz -tar -xzf nettle-3.8.1.tar.gz -cd nettle-3.8.1 -./configure --libdir=/usr/local/lib +wget https://ftp.gnu.org/gnu/nettle/nettle-3.9.1.tar.gz +tar -xzf nettle-3.9.1.tar.gz +cd nettle-3.9.1 +./configure --libdir=/usr/local/lib --enable-static --disable-shared --disable-openssl --disable-mini-gmp -disable-gcov --disable-documentation make -j $(nproc) sudo make install ``` Since Ubuntu 20.04, you need to specify the library directory explicitly. Otherwise, the library will be installed in custom locations where it would not be found by `cmake`. +## Compile `libmbedtls` from source + +*FTL*DNS uses another cryptographic library (`libmbedtls`) containing cryptographic primitives, X.509 certificate manipulation and the SSL/TLS and DTLS protocols used for serving the web interface and the API over HTTPS. + +Compile and install a recent version using: + +```bash +wget https://github.com/Mbed-TLS/mbedtls/archive/refs/tags/v3.5.0.tar.gz -O mbedtls-3.5.0.tar.gz +tar -xzf mbedtls-3.5.0.tar.gz +cd mbedtls-3.5.0 +sed -i '/#define MBEDTLS_THREADING_C/s*^//**g' include/mbedtls/mbedtls_config.h +sed -i '/#define MBEDTLS_THREADING_PTHREAD/s*^//**g' include/mbedtls/mbedtls_config.h +make -j $(nproc) +sudo make install +``` + +The `sed` commands are necessary to enable multi-threading support in `libmbedtls` as there is no `configure` script to do this for us (see also [here](https://github.com/Mbed-TLS/mbedtls#configuration)). + ## Get the source Now, clone the *FTL*DNS repo (or your own fork) to get the source code of *FTL*DNS: @@ -48,7 +66,7 @@ git clone https://github.com/pi-hole/FTL.git && cd FTL If you want to build another branch and not `master`, use checkout to get to this branch, like ```bash -git checkout development +git checkout development-v6 ``` ## Compile the source @@ -89,6 +107,10 @@ Finally, restart *FTL*DNS to use the new binary: sudo service pihole-FTL restart ``` +## Caution + +Once your homebrew `pihole-FTL` binary is built and installed, do not run `pihole -up` or `pihole checkout`. These commands might overwrite your local `pihole-FTL` binary with Pi-hole's pre-compiled binaries. + # Use containerized build environment While most people think of [Docker](https://www.docker.com/) as a deployment environment, it's also a wonderful tool to create and maintain build environments. Pi-hole provides `ftl-build` containers composed of everything needed to build FTL for various architectures on your `x86_64` hosts. Check out [Docker Hub `pi-hole/ftl-build`](https://hub.docker.com/r/pihole/ftl-build/tags) for the available build containers as well as the [Releases overview](https://github.com/pi-hole/docker-base-images/releases/) for a detailed changelog. diff --git a/docs/ftldns/configfile.md b/docs/ftldns/configfile.md index 9d8226d3c..57c4d1333 100644 --- a/docs/ftldns/configfile.md +++ b/docs/ftldns/configfile.md @@ -323,11 +323,11 @@ Print information about ARP table processing: How long did parsing take, whether Controls if *FTL*DNS should print extended details about regex matching into `FTL.log`. -**[More details](../regex/overview.md)** +**[More details](../regex/index.md)** -#### `DEBUG_API=false|true` {#debug_api data-toc-label='Telnet'} +#### `DEBUG_API=false|true` {#debug_api data-toc-label='API'} -Print extra debugging information during telnet API calls. Currently only used to send extra information when getting all queries. +Print extra debugging information during API calls. #### `DEBUG_OVERTIME=false|true` {#debug_overtime data-toc-label='Over Time Data'} diff --git a/docs/ftldns/debugging.md b/docs/ftldns/debugging.md index 9ffdf96e9..4ffb9cf2a 100644 --- a/docs/ftldns/debugging.md +++ b/docs/ftldns/debugging.md @@ -1,17 +1,47 @@ # Debugging FTLDNS using `gdb` +`gdb` is a powerful debugger that can be used to analyze and help fixing issues in software. It is a command-line tool that can be used to inspect the state of a running process, set breakpoints, and step through the code. Furthermore, it can be used to investigate crashes and give access to the state of the program (e.g. precise locations and variable contents) at the time of the crash. +This guide will show you how to use `gdb` to debug `pihole-FTL`. + Once you are used to it, you can skip most of the steps. Debugging *FTL*DNS is quite easy. `pihole-FTL` has been designed so that a debugger can be attached to an already running process. This will give you insights into how software (not limited to `pihole-FTL`) works. + +!!! info "When running Pi-hole in a Docker container" + If you are running Pi-hole in a Docker container, you will need to perform all the steps described here *inside* the Docker container. You can use, e.g., `docker exec -it /bin/bash` to get a shell inside the container. + + You will also need to (temporarily!) add the `SYS_PTRACE` capability. This is required to allow `gdb` to attach to the `pihole-FTL` process inside the container. You can do this by starting the container with the `--cap-add=SYS_PTRACE` option. Furthermore, you need to disable the Linux kernel's secure computing mode for the container. Add `--security-opt seccomp:unconfined` in order to allow `gdb` to disable address space randomization. + + ```bash + docker run -d --name pihole [your other options] --cap-add=SYS_PTRACE --security-opt seccomp:unconfined pihole/pihole + ``` + + or, when using `docker-compose`: + + ```yaml + services: + pihole: + + # your other options ... + + cap_add: + - # your other added capabilities ... + - SYS_PTRACE + + security_opt: + - seccomp:unconfined + ``` + + ## Prerequirements (only required once) -1. Install `screen` and `gdb` using `sudo apt-get install screen gdb` +1. Install `screen` and `gdb` using `sudo apt install screen gdb` (you may need to use another package manager, e.g., `yum`, `dnf` or `apk add`, depending on your distribution) 2. Start a screen session (it will allow you to come back even if the SSH connection died) * If you don't know about `screen`, then read about it (you *will* love it!) 3. Start a screen session using `screen` 4. Configure `gdb` by installing a globally valid initialization file: ```bash - echo "handle SIGHUP nostop SIGPIPE nostop SIGTERM nostop SIG32 nostop SIG34 nostop SIG35 nostop" | sudo tee /root/.gdbinit + echo "handle SIGHUP nostop SIGPIPE nostop SIGTERM nostop SIG32 nostop SIG34 nostop SIG35 nostop SIG41 nostop" | sudo tee /root/.gdbinit ``` You can omit this step, however, you will have to remember to run the quoted line on *every start* of `gdb` in order to properly debug FTL. diff --git a/docs/ftldns/dns-cache.md b/docs/ftldns/dns-cache.md deleted file mode 100644 index e2ed3bd43..000000000 --- a/docs/ftldns/dns-cache.md +++ /dev/null @@ -1,39 +0,0 @@ -`pihole-FTL` offers an efficient DNS cache that helps speed up your Internet experience. This DNS cache is part of the embedded `dnsmasq` server. Setting the cache size to zero disables caching. The DNS TTL value is used for determining the caching period. `pihole-FTL` clears its cache on receiving `SIGHUP`. - - -!!! warning Some warning about the DNS cache size - **There is no benefit in increasing this number *unless* the number of DNS cache evictions is greater than zero.** - - A larger cache *will* consume more memory on your node, leaving less memory available for other caches of your Pi-hole. If you push this number to the extremes, it may even be that your Pi-hole gets short on memory and does not operate as expected. - - You can not reduce the cache size below `150` when DNSSEC is enabled because the DNSSEC validation process uses the cache. - - -### Cache metrics - -The Settings page (System panel, FTL table) gives live information about the cache usage. It obtains its information from `http://pi.hole/admin/api.php?getCacheInfo`. - -#### DNS cache size - -Size of the DNS domain cache, defaulting to 10,000 entries. It is the number of entries that can be actively cached at the same time. -This information may also be queried using `dig +short chaos txt cachesize.bind` - -The cache size is set in `/etc/dnsmasq.d/01-pihole.conf`. However, note that this setting does not survive Pi-hole updates. If you want to change the cache size permanently, add a setting - -```plain -CACHE_SIZE=12345 -``` - -in `/etc/pihole/setupVars.conf` and run `pihole -r` (Repair) to get the cache size changed for you automatically. - -#### DNS cache insertions - -Number of total insertions into the cache. This number can be substantially larger than DNS cache size as expiring cache entries naturally make room for new insertions over time. Each lookup with a non-zero TTL will be cached. - -This information may also be queried using `dig +short chaos txt insertions.bind` - -#### DNS cache evictions - -The number of cache entries that had to be removed although the corresponding entries were **not** expired. Old cache entries get removed if the cache is full to make space for more recent domains. The cache size should be increased when this number is larger than zero. - -This information may also be queried using `dig +short chaos txt evictions.bind` diff --git a/docs/ftldns/in-depth.md b/docs/ftldns/in-depth.md deleted file mode 100644 index e021f83e8..000000000 --- a/docs/ftldns/in-depth.md +++ /dev/null @@ -1,42 +0,0 @@ -## Available interfaces - -Pi-hole stats can be accessed via a standard Unix socket (`/run/pihole/FTL.sock`), a telnet-like connection (TCP socket on port `4711`) as well as indirectly via the Web API (`admin/api.php`), and the command line (`pihole -c -j`). You can out find more details below. - -## Command-line arguments - -- `debug` - Don't go into daemon mode (stay in foreground) + more verbose logging -- `test` - Start `FTL` and process everything, but shut down immediately afterward -- `version` - Don't start `FTL`, only show the version -- `tag` - Don't start `FTL`, show only git tag -- `branch` - Don't start `FTL`, show only git branch `FTL` was compiled from -- `no-daemon` or `-f` - Don't go into background (daemon mode) -- `help` or `-h` - Don't start `FTL`, show help -- `dnsmasq-test` - Test resolver config file syntax -- `--` everything behind `--` will be passed as options to the internal resolver - -Command-line arguments can be arbitrarily combined, e.g. `pihole-FTL debug test` - -## File locations - -- `/var/log/pihole/FTL.log` log file -- `/run/pihole-FTL.pid` PID file -- `/run/pihole-FTL.port` file containing port on which `FTL` is listening -- `/run/pihole/FTL.sock` Unix socket - -## Linux capabilities - -Capabilities (POSIX 1003.1e, [capabilities(7)](https://man7.org/linux/man-pages/man7/capabilities.7.html)) provide fine-grained control over superuser permissions, allowing the use of the `root` user to be avoided. -To perform permission checks, traditional UNIX implementations distinguish two categories of processes: *privileged processes* (superuser or `root`), and *unprivileged processes*. Privileged processes bypass all kernel permission checks, while unprivileged processes are subject to full permission checking based on the process's credentials (user and group permissions and supplementary process capabilities). Capabilities are implemented on Linux using extended attributes ([xattr(7)](https://man7.org/linux/man-pages/man7/xattr.7.html)) in the `security` namespace. Extended attributes are supported by all major Linux file systems, including Ext2, Ext3, Ext4, Btrfs, JFS, XFS, and ReiserFS. - -For your safety and comfort, `pihole-FTL` is run by the entirely unprivileged user `pihole`. -Whereas `dnsmasq` is running as `root` process, we designed `pihole-FTL` to be run by the entirely unprivileged user `pihole`. As a consequence, `pihole-FTL` will not be able to access the files of any other user on this system or mess around with your system's configuration. - -However, this also implies that *FTL*DNS cannot bind to ports 53 (DNS) among some other necessary capabilities related to DHCP services. To establish a strong security model, we explicitly grant the `pihole-FTL` process additional capabilities so that `pihole-FTL` (but no other processes which may be started by `pihole`) can bind to port 53, etc., without giving any additional permissions to the `pihole` user. - -We specifically add the following capabilities to `pihole-FTL`: - -- `CAP_NET_BIND_SERVICE`: Allows *FTL*DNS binding to TCP/UDP sockets below 1024 (specifically DNS service on port 53) -- `CAP_NET_RAW`: use raw and packet sockets (we need a RAW socket for handling DHCPv6 requests) -- `CAP_NET_ADMIN`: modify routing tables and other network-related operations (to allow for handling DHCP requests) - -Users that cannot use Linux capabilities for various reasons (lacking kernel or file system support) can modify the startup scripts of `pihole-FTL` to ensure the daemon is started as `root`. However, be aware that you do so on your own risk (although we don't expect problems to arise). diff --git a/docs/ftldns/signals.md b/docs/ftldns/signals.md index 65331ac2a..b7f7f24d2 100644 --- a/docs/ftldns/signals.md +++ b/docs/ftldns/signals.md @@ -74,3 +74,11 @@ Re-resolve all clients and forward destination hostnames. This forces refreshing ## Real-time signal 5 (SIG39) Re-parse ARP/neighbour-cache now to update the Network table now + +## Real-time signal 6 (SIG40) + +Signal used internally to terminate the embedded `dnsmasq`. Please do not use this signal to prevent misbehaviour. + +## Real-time signal 7 (SIG41) + +Scan binary search lookup tables for hash collisions and report if any are found. This is a debugging signal and not meaningful production. Scanning the lookup tables is a time-consuming operation and may stall DNS resolution for a while on low-end devices. diff --git a/docs/ftldns/telnet-api.md b/docs/ftldns/telnet-api.md deleted file mode 100644 index a55d14d2f..000000000 --- a/docs/ftldns/telnet-api.md +++ /dev/null @@ -1,268 +0,0 @@ -Connect via e.g. `telnet 127.0.0.1 4711` or use `echo ">command" | nc 127.0.0.1 4711` - -#### `>quit` {data-toc-label='quit'} - -Closes the connection to the client - ---- - -#### `>stats` {data-toc-label='stats'} - -Get current statistics - -```text -domains_being_blocked 116007 -dns_queries_today 30163 -ads_blocked_today 5650 -ads_percentage_today 18.731558 -unique_domains 1056 -queries_forwarded 4275 -queries_cached 20238 -clients_ever_seen 11 -unique_clients 9 -status enabled -``` - ---- - -#### `>overTime` {data-toc-label='overTime'} - -Get over time data (10 min intervals) - -```text -1525546500 163 0 -1525547100 154 1 -1525547700 164 0 -1525548300 167 0 -1525548900 151 0 -1525549500 143 0 -[...] -``` - ---- - -#### `>top-domains` {data-toc-label='top-domains'} - -Get top domains - -```text -0 8462 x.y.z.de -1 236 safebrowsing-cache.google.com -2 116 pi.hole -3 109 z.y.x.de -4 93 safebrowsing.google.com -5 96 plus.google.com -[...] -``` - -Variant: `>top-domains (15)` to show (up to) 15 entries - ---- - -#### `>top-ads` {data-toc-label='top-ads'} - -Get top ad domains - -```text -0 8 googleads.g.doubleclick.net -1 6 www.googleadservices.com -2 1 cdn.mxpnl.com -3 1 collector.githubapp.com -4 1 www.googletagmanager.com -5 1 s.zkcdn.net -[...] -``` - -Variant: `>top-ads (14)` to show (up to) 14 entries - ---- - -#### `>top-clients` {data-toc-label='top-clients'} - -Get recently active top clients (IP addresses + hostnames (if available)) - -```text -0 9373 192.168.2.1 router -1 484 192.168.2.2 work-machine -2 8 127.0.0.1 localhost -``` - -Variant: `>top-clients (9)` to show (up to) 9 client entries or `>top-clients withzero (15)` to show (up to) 15 clients even if they have not been active recently (see PR #124 for further details) - ---- - -#### `>forward-dest` {data-toc-label='forward-dest'} - -Get forward destinations (IP addresses + hostnames (if available)) along with the percentage. The first result (ID -2) will always be the percentage of domains answered from blocklists, whereas the second result (ID -1) will be the queries answered from the cache - -```text --2 18.70 blocklist blocklist --1 67.10 cache cache -0 14.20 127.0.0.1 localhost -``` - -Variant: `>forward-dest unsorted` to show forward destinations in unsorted order (equivalent to using `>forward-names`) - ---- - -#### `>querytypes` {data-toc-label='querytypes'} - -Get collected query types percentage - -```text -A (IPv4): 53.45 -AAAA (IPv6): 45.32 -ANY: 0.00 -SRV: 0.64 -SOA: 0.05 -PTR: 0.54 -TXT: 0.00 -``` - ---- - -#### `>getallqueries` {data-toc-label='getallqueries'} - -Get all queries that FTL has in memory - -```text -1525554586 A fonts.googleapis.com 192.168.2.100 3 0 4 6 -1525554586 AAAA fonts.googleapis.com 192.168.2.100 3 0 4 5 -1525554586 A www.mkdocs.org 192.168.2.100 3 0 4 7 -1525554586 AAAA www.mkdocs.org 192.168.2.100 2 0 3 21 -1525554586 A squidfunk.github.io 192.168.2.100 2 0 3 20 -1525554586 A pi-hole.net 192.168.2.100 3 0 4 5 -1525554586 AAAA squidfunk.github.io 192.168.2.100 3 0 1 6 -1525554586 AAAA pi-hole.net 192.168.2.100 2 0 1 18 -1525554586 A github.com 192.168.2.100 3 0 4 5 -1525554586 AAAA github.com 192.168.2.100 2 0 1 18 -``` - -Variants: - -- `>getallqueries (37)` show (up to) 37 latest entries, -- `>getallqueries-time 1483964295 1483964312` gets all queries that FTL has in its database in a limited time interval, -- `>getallqueries-time 1483964295 1483964312 (17)` show matches in the (up to) 17 latest entries, -- `>getallqueries-domain www.google.com` gets all queries that FTL has in its database for a specific domain name, -- `>getallqueries-client 2.3.4.5`: gets all queries that FTL has in its database for a specific client name *or* IP - ---- - -#### `>recentBlocked` {data-toc-label='recentBlocked'} - -Get most recently pi-holed domain name - -```text -www.googleadservices.com -``` - -Variant: `>recentBlocked (4)` show the four most recent blocked domains - ---- - -#### `>clientID` {data-toc-label='clientID'} - -Get ID of currently connected client - -```text -6 -``` - ---- - -#### `>version` {data-toc-label='version'} - -Get version information of the currently running FTL instance - -```text -version v1.6-3-g106498d-dirty -tag v1.6 -branch master -hash 106498d -date 2017-03-26 13:10:43 +0200 -``` - ---- - -#### `>dbstats` {data-toc-label='dbstats'} - -Get some statistics about `FTL`'s' long-term storage database (this request may take some time for processing in case of a large database file) - -```text -queries in database: 2700304 -database filesize: 199.20 MB -SQLite version: 3.23.1 -``` - ---- - -#### `>cacheinfo` {data-toc-label='cacheinfo'} - -Get DNS server cache size and usage information - -```text -cache-size: 500000 -cache-live-freed: 0 -cache-inserted: 15529 -``` - ---- - -#### `>dns-port` {data-toc-label='dns-port'} - -Get DNS port FTL is listening on - -```text -53 -``` - -Note that the port can also be `0` if someone decides to disable the DNS server part of Pi-hole - ---- - -#### `>maxlogage` {data-toc-label='maxlogage'} - -Get timespan of the statistics shown on the dashboard (in seconds) - -```text -86400 -``` - ---- - -#### `>gateway` {data-toc-label='gateway'} - -Get the IP of the gateway of the default route and the corresponding interface - -```text -192.168.0.1 enp2s0 -``` - -Note that if no non-default route could be found, `0.0.0.0` and an empty interface string is returned - ---- - -#### `>interfaces` {data-toc-label='interfaces'} - -Get extended information of the interfaces of th Pi-hole device - -```text -eth0 UP 1000 2.2GB 5.6GB 10.0.1.5 fd00:e57b:XXXX:210e:1a1,2a01:XXXX:c15b,fe80::2e5c:XXXX:4060 -wlan0 DOWN -1 0.0B 0.0B - - -docker0 UP 10000 837.6MB 300.7MB 172.17.0.1,169.254.241.237 - -lo UP -1 48.0MB 48.0MB 127.0.0.1 - -wg0 UP -1 1.0GB 141.2MB 10.0.40.1 - -sum UP 0 4.2GB 6.1GB - - -``` - -Column definitions are: - -1. Interface name -2. UP/DOWN status -3. Link speed in MBit/s (-1 means "Not available" (like link down) or "Not applicable" (like virtual interface)) -4. TX bytes -5. RX bytes -6. Associated IPv4 addresses -7. Associated IPv6 addresses - -The default interface (the one connected to the gateway) will always be the first. The sum will always be the last one - even if you have (for whatever reason) an interface called sum. Regarding the link speed: It won't work for most WiFi interfaces as the speed is not known at the kernel level. Instead, the drivers manage them dynamically depending on package loss, signal strength, etc. - in this case, you'll see link speed -1 as well diff --git a/docs/group_management/groups.md b/docs/group_management/index.md similarity index 97% rename from docs/group_management/groups.md rename to docs/group_management/index.md index 07e697c3b..73c06b52b 100644 --- a/docs/group_management/groups.md +++ b/docs/group_management/index.md @@ -1,5 +1,3 @@ -## Group management - Any blocklist or domain on the white-/black-/regex-lists can be managed through groups. This allows not only grouping them to highlight their relationship, but also enabling/disabling them together if one, for instance, wants to visit a specific service only temporarily. Group `Default` (`group_id` `0`) is special as it is automatically assigned to domains and clients not being a member of other groups. Each newly added client or domain gets assigned to group zero when being added. diff --git a/docs/guides/github/contributing.md b/docs/guides/github/index.md similarity index 100% rename from docs/guides/github/contributing.md rename to docs/guides/github/index.md diff --git a/docs/guides/misc/tor/overview.md b/docs/guides/misc/tor/index.md similarity index 100% rename from docs/guides/misc/tor/overview.md rename to docs/guides/misc/tor/index.md diff --git a/docs/guides/vpn/openvpn/deprecation_notice.md b/docs/guides/vpn/openvpn/deprecation_notice.md index 02b2d0737..758e49e5c 100644 --- a/docs/guides/vpn/openvpn/deprecation_notice.md +++ b/docs/guides/vpn/openvpn/deprecation_notice.md @@ -2,4 +2,4 @@ !!! warning "OpenVPN is no longer recommended" We do no longer recommending the use of OpenVPN for new deployments. Although OpenVPN has served us well in the past, we believe it's time to move towards more modern and efficient solutions. - We suggest that users now turn their attention to [WireGuard](../wireguard/overview.md), a forward-thinking VPN solution that offers better performance, faster speeds, and easier implementation. WireGuard has been designed with the latest technology in mind, providing simple yet powerful tools for securing your network communications. Pi-hole's step-by-step tutorial is designed to help you understand the ins and outs of WireGuard, regardless of your technical expertise. + We suggest that users now turn their attention to [WireGuard](../wireguard/index.md), a forward-thinking VPN solution that offers better performance, faster speeds, and easier implementation. WireGuard has been designed with the latest technology in mind, providing simple yet powerful tools for securing your network communications. Pi-hole's step-by-step tutorial is designed to help you understand the ins and outs of WireGuard, regardless of your technical expertise. diff --git a/docs/guides/vpn/openvpn/overview.md b/docs/guides/vpn/openvpn/index.md similarity index 100% rename from docs/guides/vpn/openvpn/overview.md rename to docs/guides/vpn/openvpn/index.md diff --git a/docs/guides/vpn/wireguard/overview.md b/docs/guides/vpn/wireguard/index.md similarity index 100% rename from docs/guides/vpn/wireguard/overview.md rename to docs/guides/vpn/wireguard/index.md diff --git a/docs/images/api/android-ca-name.png b/docs/images/api/android-ca-name.png new file mode 100644 index 000000000..4f480b9a3 Binary files /dev/null and b/docs/images/api/android-ca-name.png differ diff --git a/docs/images/api/android-ca-security-warning.png b/docs/images/api/android-ca-security-warning.png new file mode 100644 index 000000000..3800e2aba Binary files /dev/null and b/docs/images/api/android-ca-security-warning.png differ diff --git a/docs/images/api/android-ca-select.png b/docs/images/api/android-ca-select.png new file mode 100644 index 000000000..f499c8ce4 Binary files /dev/null and b/docs/images/api/android-ca-select.png differ diff --git a/docs/images/api/android-ca-success.png b/docs/images/api/android-ca-success.png new file mode 100644 index 000000000..b6ea67ab9 Binary files /dev/null and b/docs/images/api/android-ca-success.png differ diff --git a/docs/images/api/android-chrome-trusted.png b/docs/images/api/android-chrome-trusted.png new file mode 100644 index 000000000..73155cdb7 Binary files /dev/null and b/docs/images/api/android-chrome-trusted.png differ diff --git a/docs/images/api/android-chrome-untrusted.png b/docs/images/api/android-chrome-untrusted.png new file mode 100644 index 000000000..384f85360 Binary files /dev/null and b/docs/images/api/android-chrome-untrusted.png differ diff --git a/docs/images/api/android-credential-storage.png b/docs/images/api/android-credential-storage.png new file mode 100644 index 000000000..0e1be7f86 Binary files /dev/null and b/docs/images/api/android-credential-storage.png differ diff --git a/docs/images/api/android-install-ca.png b/docs/images/api/android-install-ca.png new file mode 100644 index 000000000..2aaa1c0ba Binary files /dev/null and b/docs/images/api/android-install-ca.png differ diff --git a/docs/images/api/android-pihole-trusted.png b/docs/images/api/android-pihole-trusted.png new file mode 100644 index 000000000..ca45cdccc Binary files /dev/null and b/docs/images/api/android-pihole-trusted.png differ diff --git a/docs/images/api/android-pihole-untrusted.png b/docs/images/api/android-pihole-untrusted.png new file mode 100644 index 000000000..108b304da Binary files /dev/null and b/docs/images/api/android-pihole-untrusted.png differ diff --git a/docs/images/api/android-system-security.png b/docs/images/api/android-system-security.png new file mode 100644 index 000000000..eaa446174 Binary files /dev/null and b/docs/images/api/android-system-security.png differ diff --git a/docs/images/api/chrome-ca-added.png b/docs/images/api/chrome-ca-added.png new file mode 100644 index 000000000..d25e5728b Binary files /dev/null and b/docs/images/api/chrome-ca-added.png differ diff --git a/docs/images/api/chrome-ca-trust.png b/docs/images/api/chrome-ca-trust.png new file mode 100644 index 000000000..66842327d Binary files /dev/null and b/docs/images/api/chrome-ca-trust.png differ diff --git a/docs/images/api/chrome-certificates.png b/docs/images/api/chrome-certificates.png new file mode 100644 index 000000000..62f13e6ba Binary files /dev/null and b/docs/images/api/chrome-certificates.png differ diff --git a/docs/images/api/chrome-pihole-trusted.png b/docs/images/api/chrome-pihole-trusted.png new file mode 100644 index 000000000..d8bf6e3b6 Binary files /dev/null and b/docs/images/api/chrome-pihole-trusted.png differ diff --git a/docs/images/api/chrome-pihole-untrusted.png b/docs/images/api/chrome-pihole-untrusted.png new file mode 100644 index 000000000..4c1384bdb Binary files /dev/null and b/docs/images/api/chrome-pihole-untrusted.png differ diff --git a/docs/images/api/firefox-ca-added.png b/docs/images/api/firefox-ca-added.png new file mode 100644 index 000000000..6bd74f96c Binary files /dev/null and b/docs/images/api/firefox-ca-added.png differ diff --git a/docs/images/api/firefox-ca-trust.png b/docs/images/api/firefox-ca-trust.png new file mode 100644 index 000000000..d66fc71b7 Binary files /dev/null and b/docs/images/api/firefox-ca-trust.png differ diff --git a/docs/images/api/firefox-certificates.png b/docs/images/api/firefox-certificates.png new file mode 100644 index 000000000..164ff35b3 Binary files /dev/null and b/docs/images/api/firefox-certificates.png differ diff --git a/docs/images/api/firefox-pihole-trusted.png b/docs/images/api/firefox-pihole-trusted.png new file mode 100644 index 000000000..1c03b6fb5 Binary files /dev/null and b/docs/images/api/firefox-pihole-trusted.png differ diff --git a/docs/images/api/firefox-pihole-untrusted.png b/docs/images/api/firefox-pihole-untrusted.png new file mode 100644 index 000000000..98138487f Binary files /dev/null and b/docs/images/api/firefox-pihole-untrusted.png differ diff --git a/docs/images/api/firefox-tls-insecure.png b/docs/images/api/firefox-tls-insecure.png new file mode 100644 index 000000000..aa777d0b7 Binary files /dev/null and b/docs/images/api/firefox-tls-insecure.png differ diff --git a/docs/main/index.md b/docs/main/index.md new file mode 100644 index 000000000..70725f1e2 --- /dev/null +++ b/docs/main/index.md @@ -0,0 +1,11 @@ +Before installing Pi-hole, you need to make sure your system meets the [prerequisites](prerequisites.md) (hardware and operating system) and decide whether you want a [normal installation](basic-install.md) (bare metal) or a [docker installation](https://github.com/pi-hole/docker-pi-hole/). + +After the installation is complete, check the [Post-Install steps](post-install.md) to make sure your network is using Pi-hole. + +With the [`pihole` command](pihole-command.md), you can manually run Pi-hole commands, including [update Pi-hole](update.md). + +Alternatively, you can also use the [web interface](https://github.com/pi-hole/web/) to run and configure your Pi-hole. + +The web interface also allows you to use [Group Management](../group_management/index.md) functions to manage the relationship between clients, blocking rules, and allowing rules. + +If you need help, please visit our [Discourse Forum](https://discourse.pi-hole.net/c/bugs-problems-issues/11). diff --git a/docs/core/pihole-command.md b/docs/main/pihole-command.md similarity index 71% rename from docs/core/pihole-command.md rename to docs/main/pihole-command.md index 14d6c5d70..1a0a31aa5 100644 --- a/docs/core/pihole-command.md +++ b/docs/main/pihole-command.md @@ -1,13 +1,8 @@ ---- -title: The pihole command - Pi-hole documentation ---- - Pi-hole makes use of many commands, and here we will break down those required to administer the program via the command-line Interface. | Index | Invocation | | -------------- | -------------- | | [Core Script](#pi-hole-core) | `pihole` | -| [Web Script](#pi-hole-web) | `pihole -a` | --- @@ -21,7 +16,7 @@ Pi-hole makes use of many commands, and here we will break down those required t | [Log Flush](#log-flush) | `pihole flush` | | [Reconfigure](#reconfigure) | `pihole reconfigure` | | [Tail](#tail) | `pihole tail` | -| [Admin](#admin) | `pihole -a` | +| [Set password](#password) | `pihole setpassword` | | [Gravity](#gravity) | `pihole updateGravity` | | [Logging](#logging) | `pihole logging` | | [Query](#query) | `pihole query` | @@ -53,12 +48,12 @@ The core script of Pi-hole provides the ability to tie many DNS related function Administrators need to be able to manually add and remove domains for various purposes, and these commands serve that purpose. -See [Regex Blocking](../regex/overview.md) for more information about using Regex. +See [Regex Blocking](../regex/index.md) for more information about using Regex. **Basic Script Process**: * Each domain is validated using regex (except when using `--regex`), to ensure invalid domains and IDNs are not added -* A domain gets added to or removed from the `domainlist` table in [`/etc/pihole/gravity.db`](../database/gravity/index.md) +* A domain gets added to or removed from the `domainlist` table in [`/etc/pihole/gravity.db`](../database/domain-database/index.md) * The DNS server is then reloaded ### Debugger @@ -108,34 +103,6 @@ There are times where the administrator will need to repair or reconfigure the P Since Pi-hole will log DNS queries by default, using this command to watch the log in real-time can be useful for debugging a problematic site, or even just for sheer curiosities sake. -### Admin - -| | | -| -------------- | -------------- | -| Help Command | `pihole -a --help` | -| Script Location | [`/opt/pihole/webpage.sh`](https://github.com/pi-hole/pi-hole/blob/master/advanced/Scripts/webpage.sh) | -| Example Usage | `pihole -a -p secretpassword` | - -Detailed information on this is [found here](#web-script). - -### Chronometer - -| | | -| -------------- | -------------- | -| Help Command | `pihole -c --help` | -| Script Location | [`/opt/pihole/chronometer.sh`](https://github.com/pi-hole/pi-hole/blob/master/advanced/Scripts/chronometer.sh) | -| Example Usage | [`pihole -c -e`](https://discourse.pi-hole.net/t/the-pihole-command-with-examples/738#chronometer) | - -Chronometer is a console dashboard of real-time stats, which can be displayed via `ssh` or on an LCD screen attached directly to your hardware. The script is capable of detecting the size of your screen and adjusting output to try and best suit it. - - - Chronometer in action - -
- - Image courtesy of /u/super_nicktendo22 - - ### Gravity | | | @@ -149,11 +116,11 @@ Gravity is one of the most important scripts of Pi-hole. Its main purpose is to **Basic Script Process**: * It will determine Internet connectivity, and give time for `pihole-FTL` to be resolvable on low-end systems if has just been restarted -* It extracts all URLs and domains from the `adlists` table in [`/etc/pihole/gravity.db`](../database/gravity/index.md) +* It extracts all URLs and domains from the `adlists` table in [`/etc/pihole/gravity.db`](../database/domain-database/index.md) * It runs through each URL, downloading it if necessary * `curl` checks the servers `Last-Modified` header to ensure it is getting a newer version * It will attempt to parse the file into a domains-only format if necessary -* Lists are merged, comments removed, sorted uniquely and stored in the `gravity` table of [`/etc/pihole/gravity.db`](../database/gravity/index.md) +* Lists are merged, comments removed, sorted uniquely and stored in the `gravity` table of [`/etc/pihole/gravity.db`](../database/domain-database/index.md) * Gravity cleans up temporary content and reloads the DNS server ### Logging @@ -180,7 +147,7 @@ This command will query your whitelist, blacklist, wildcards and adlists for a s * User-specified options are handled * Using `idn`, it will convert [Internationalized domain names](https://en.wikipedia.org/wiki/Internationalized_domain_name) into [punycode](https://en.wikipedia.org/wiki/Punycode) -* Database at [`/etc/pihole/gravity.db`](../database/gravity/index.md) is queried to return a list of adlists in which the queried domain exists. +* Database at [`/etc/pihole/gravity.db`](../database/domain-database/index.md) is queried to return a list of adlists in which the queried domain exists. ### Update @@ -260,62 +227,12 @@ Switch Pi-hole subsystems to a different GitHub branch. An admin can specify rep --- -## Pi-hole Web - -| Feature | Invocation | -| -------------- | -------------- | -| [Web Script](#web-script) | `pihole -a` | -| [Password](#password) | `pihole -a password` | -| [Teleport](#teleport) | `pihole -a -t` | -| [Temperature Unit](#temperature-unit) | `pihole -a celsius`, `pihole -a fahrenheit`, `pihole -a kelvin` | -| [Interface](#interface) | `pihole -a interface` | - -### Web Script - -| | | -| -------------- | -------------- | -| Help Command | `pihole -a --help` | -| Script Location | [`/opt/pihole/webpage.sh`](https://github.com/pi-hole/pi-hole/blob/master/advanced/Scripts/webpage.sh) | -| Example Usage | `pihole -a -p secretpassword` | - -Set options for the Web Interface. This script is used to tie in all Web Interface features which are not already covered by the [Core Script](#core-script). - ### Password | | | | -------------- | -------------- | | Help Command | N/A | -| Script Location | [`/opt/pihole/webpage.sh`](https://github.com/pi-hole/pi-hole/blob/master/advanced/Scripts/webpage.sh) | -| Example Usage | [`pihole -a -p secretpassword`](https://discourse.pi-hole.net/t/the-pihole-command-with-examples/738#web-password) | - -Set the Web Interface password. Password can be entered as an option (e.g: `pihole -a -p secretpassword`), or separately as to not display on the screen (e.g: `pihole -a -p`). - -### Teleport - -| | | -| -------------- | -------------- | -| Help Command | N/A | -| Script Location | N/A | -| Example Usage | `pihole -a -t` | - -Create a configuration backup. The backup will be created in the directory from which the command is run. The backup can be imported using the Settings > Teleport page. - -### Temperature Unit - -| | | -| -------------- | -------------- | -| Help Command | N/A | -| Script Location | [`/opt/pihole/webpage.sh`](https://github.com/pi-hole/pi-hole/blob/master/advanced/Scripts/webpage.sh) | -| Example Usage | [`pihole -a -c`](https://discourse.pi-hole.net/t/the-pihole-command-with-examples/738#temp-unit) | - -Set the specified temperature unit as the preferred type. This preference will affect the Web Interface, as well as Chronometer. - -### Interface - -| | | -| -------------- | -------------- | -| Help Command | `pihole -a interface --help` | -| Script Location | [`/opt/pihole/webpage.sh`](https://github.com/pi-hole/pi-hole/blob/master/advanced/Scripts/webpage.sh) | -| Example Usage | [`pihole -a interface local`](https://discourse.pi-hole.net/t/the-pihole-command-with-examples/738#interface) | +| Script Location | [`/usr/local/bin/pihole`](https://github.com/pi-hole/pi-hole/blob/master/pihole) | +| Example Usage | `pihole setpassword` | -Specify interface listening behavior for `pihole-FTL`. When using `pihole -a interface all`, please ensure you use a firewall to prevent your Pi-hole from becoming an unwitting host to [DNS amplification attackers](https://duckduckgo.com/?q=dns+amplification+attack). You may want to consider running [Wireguard](../guides/vpn/wireguard/overview.md) to grant your mobile devices access to the Pi-hole. +Set the Web Interface an API password. Password can be entered as an option (e.g: `pihole setpassword secretpassword`), or separately as to not display on the screen (e.g: `pihole setpassword`). diff --git a/docs/main/prerequisites.md b/docs/main/prerequisites.md index 0ea91c674..83dba45ff 100644 --- a/docs/main/prerequisites.md +++ b/docs/main/prerequisites.md @@ -69,8 +69,6 @@ Pi-hole only supports actively maintained versions of these systems. Pi-hole needs a static IP address to properly function (a DHCP reservation is just fine). -On systems that have `dhcpcd5` already installed (e.g Raspberry Pi OS) there is an option in the install process to append some lines to `/etc/dhcpcd.conf` in order to statically assign an IP address. This is an entirely optional step, and offered as a way to lower the barrier to entry for those that may not be familiar with linux systems, such as those first starting out on a Raspberry Pi. - ### Ports | Service | Port | Protocol | Notes | diff --git a/docs/regex/overview.md b/docs/regex/index.md similarity index 97% rename from docs/regex/overview.md rename to docs/regex/index.md index 70bc19d5f..206c99f55 100644 --- a/docs/regex/overview.md +++ b/docs/regex/index.md @@ -13,7 +13,7 @@ There are two locations where regex filters are important: ## How to use regular expressions for filtering domains -*FTL*DNS reads in regular expression filters from the two [`regex` database views](../database/gravity/index.md). +*FTL*DNS reads in regular expression filters from the two [`regex` database views](../database/domain-database/index.md). To tell *FTL*DNS to reload the list of regex filters, either: - Execute `pihole restartdns reload-lists` or diff --git a/docs/regex/pi-hole.md b/docs/regex/pi-hole.md index f079b80ab..9cca712b2 100644 --- a/docs/regex/pi-hole.md +++ b/docs/regex/pi-hole.md @@ -2,7 +2,7 @@ ## Only match specific query types -You can amend the regular expressions by special keywords added at the end to fine-tune regular expressions to match only specific [query types](../database/ftl.md#supported-query-types). In contrast to the description of `OTHER` as being deprecated for storing queries in the database, it is still supported for regular expressions and will match all queries that are not *explicitly* covered by the other query types (see also example below). +You can amend the regular expressions by special keywords added at the end to fine-tune regular expressions to match only specific [query types](../database/query-database.md#supported-query-types). In contrast to the description of `OTHER` as being deprecated for storing queries in the database, it is still supported for regular expressions and will match all queries that are not *explicitly* covered by the other query types (see also example below). Example: diff --git a/mkdocs.yml b/mkdocs.yml index 54e715948..c8eea4e42 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -21,6 +21,7 @@ theme: features: - navigation.top - navigation.instant + - navigation.indexes - search.suggest - search.highlight - search.share @@ -47,9 +48,9 @@ theme: name: Switch to light mode markdown_extensions: - # Code highlighting in ``` ``` blocks - # https://squidfunk.github.io/mkdocs-material/setup/extensions/python-markdown-extensions/#highlight + # Code highlighting in ``` ``` blocks, superseeds codehilite - pymdownx.highlight + # allows for the nesting of code blocks inside other blocks - pymdownx.superfences - pymdownx.inlinehilite # Table of Contents @@ -92,60 +93,81 @@ markdown_extensions: # https://squidfunk.github.io/mkdocs-material/reference/tooltips/#adding-a-glossary auto_append: - docs/abbreviations.md + # Include files in other documents like {!some/dir/in/docs/filename.md!} + - markdown_include.include: + base_path: docs + # Metadata support in pages + # (https://squidfunk.github.io/mkdocs-material/extensions/metadata/) + - meta + # Tabbed provides a syntax to easily add tabbed Markdown content. + # (https://facelessuser.github.io/pymdown-extensions/extensions/tabbed/) + - pymdownx.tabbed + # InlineHilite is an inline code highlighter inspired by CodeHilite. + # (https://facelessuser.github.io/pymdown-extensions/extensions/inlinehilite/) + - pymdownx.inlinehilite + - pymdownx.arithmatex: + generic: true nav: - - Overview: index.md - 'About Pi-hole': + - 'About Pi-hole': index.md - 'Contact Us': main/contact.md - 'Pi-hole Origins': main/origins.md - 'On the Web': main/coverage.md - 'Getting Started': + - 'Getting Started': main/index.md - 'Prerequisites': main/prerequisites.md - 'Installation': main/basic-install.md - 'Post-Install': main/post-install.md - 'Updating': main/update.md - 'Uninstalling': main/uninstall.md - - 'Pi-hole Core': - - 'The  pihole  command': core/pihole-command.md + - 'The  pihole  Command': main/pihole-command.md - 'Databases': - - 'Overview': database/index.md - - 'Query database': database/ftl.md - - 'Domain database': - - 'Overview': database/gravity/index.md - - 'Group management': database/gravity/groups.md - - 'Database recovery': database/gravity/recovery.md + - 'Databases': database/index.md + - 'Query Database': database/query-database.md + - 'Domain Database': + - 'Domain Database': database/domain-database/index.md + - 'Group Management': database/domain-database/groups.md + - 'Database Recovery': database/domain-database/recovery.md + - 'Pi-hole API': + - 'Pi-hole API': api/index.md + - 'Authentication': api/auth.md + - 'TLS/SSL': api/tls.md - 'FTLDNS': - - 'Overview': ftldns/index.md + - 'FTLDNS': ftldns/index.md - 'Configuration': ftldns/configfile.md - 'Interfaces': ftldns/interfaces.md - 'DNS resolver': ftldns/dns-resolver.md - - 'DNS cache': ftldns/dns-cache.md - 'Blocking mode': ftldns/blockingmode.md - 'Privacy levels': ftldns/privacylevels.md - 'dnsmasq warnings': ftldns/dnsmasq_warn.md - 'Advanced': - 'Install from source': ftldns/compile.md - - 'Telnet API': ftldns/telnet-api.md - 'Signals': 'ftldns/signals.md' - 'Cache dump': ftldns/cache_dump.md - 'Packet dump': ftldns/package_dump.md - 'Debugging': - 'gdb': ftldns/debugging.md - 'valgrind': ftldns/valgrind.md - - 'In-depth manual': ftldns/in-depth.md - - 'Group management': - - 'Overview': group_management/groups.md + - 'Group Management': + - 'Group Management': group_management/index.md - 'Examples': group_management/example.md - - 'RegEx blocking': - - "Overview": regex/overview.md + - 'RegEx Blocking': + - "Regex Blocking": regex/index.md - "Testing": regex/testmode.md - "Tutorial": regex/tutorial.md - "Pi-hole extensions": regex/pi-hole.md - "Approximate matching": regex/approximate.md - 'Docker': + - 'Docker': docker/index.md + - 'Configuration': docker/configuration.md + - 'Upgrading': + - 'Upgrading': docker/upgrading/index.md + - 'Upgrading from v5.x': docker/upgrading/v5-v6.md + - 'Building': docker/build-image.md - 'DHCP': docker/DHCP.md - 'Contributing': - - 'Contributing on GitHub': guides/github/contributing.md + - 'Contributing': guides/github/index.md - 'Developer Certificate of Origin (DCO)': guides/github/dco.md - 'How to sign-off commits': guides/github/how-to-signoff.md - 'How to fork and rebase': guides/github/how-to-fork-rebase.md @@ -156,7 +178,7 @@ nav: - 'Upstream DNS Providers': guides/dns/upstream-dns-providers.md - 'VPN': - 'WireGuard': - - 'Overview': guides/vpn/wireguard/overview.md + - 'Wireguard': guides/vpn/wireguard/index.md - 'Concept': guides/vpn/wireguard/concept.md - 'Install server': guides/vpn/wireguard/server.md - 'Add client(s)': guides/vpn/wireguard/client.md @@ -165,7 +187,7 @@ nav: - 'Tunnel all Internet traffic': guides/vpn/wireguard/route-everything.md - 'Troubleshooting': guides/vpn/wireguard/faq.md - 'OpenVPN': - - 'Overview': guides/vpn/openvpn/overview.md + - 'OpenVPN': guides/vpn/openvpn/index.md - 'Installation': guides/vpn/openvpn/installation.md - 'Setup OpenVPN Server': guides/vpn/openvpn/setup-openvpn-server.md - 'Firewall Configuration': guides/vpn/openvpn/firewall.md @@ -180,7 +202,7 @@ nav: - 'Misc': - 'Benchmarking': guides/misc/benchmark.md - 'Tor & Pi-hole': - - 'Overview': guides/misc/tor/overview.md + - 'Tor & Pi-hole': guides/misc/tor/index.md - 'Basic Setup': guides/misc/tor/setup.md - 'Using Tor': guides/misc/tor/using-tor.md - 'Performance and other issues': guides/misc/tor/performance-issues.md @@ -219,12 +241,13 @@ plugins: - redirects: redirect_maps: 'ftldns/database.md': database/index.md + 'ftldns/regex/index.md': ftldns/regex/overview.md 'main/presentations.md': index.md 'main/prerequesites.md': main/prerequisites.md 'guides/unbound.md': guides/dns/unbound.md 'guides/upstream-dns-providers.md': guides/dns/upstream-dns-providers.md 'guides/dns-over-https.md': guides/dns/cloudflared.md - 'guides/vpn/overview.md': guides/vpn/openvpn/overview.md + 'guides/vpn/overview.md': guides/vpn/openvpn/index.md 'guides/vpn/installation.md': guides/vpn/openvpn/installation.md 'guides/vpn/setup-openvpn-server.md': guides/vpn/openvpn/setup-openvpn-server.md 'guides/vpn/firewall.md': guides/vpn/openvpn/firewall.md @@ -236,7 +259,7 @@ plugins: 'guides/vpn/dynDNS.md': guides/vpn/openvpn/dynDNS.md 'guides/vpn/troubleshooting.md': guides/vpn/openvpn/troubleshooting.md 'guides/benchmark.md': guides/misc/benchmark.md - 'guides/tor/overview.md': guides/misc/tor/overview.md + 'guides/tor/overview.md': guides/misc/tor/index.md 'guides/tor/setup.md': guides/misc/tor/setup.md 'guides/tor/using-tor.md': guides/misc/tor/using-tor.md 'guides/tor/performance-issues.md': guides/misc/tor/performance-issues.md @@ -245,7 +268,18 @@ plugins: 'guides/nginx-configuration.md': 'guides/webserver/nginx.md' 'guides/caddy-configuration.md': 'guides/webserver/caddy.md' 'guides/traefik-configuration-nodocker.md': 'guides/webserver/traefik-nodocker.md' - 'ftldns/regex/index.md': regex/overview.md - 'ftldns/regex/overview.md': regex/overview.md + 'ftldns/regex/index.md': regex/index.md + 'ftldns/regex/overview.md': regex/index.md 'ftldns/regex/tutorial.md': regex/tutorial.md 'database/gravity/example.md': group_management/example.md + 'core/pihole-command.md': main/pihole-command.md + 'database/ftl.md': database/query-database.md + 'database/gravity/index.md': database/domain-database/index.md + 'database/gravity/recovery.md': database/domain-database/recovery.md + 'database/gravity/groups.md': database/domain-database/groups.md + 'group_management/groups.md': group_management/index.md + 'regex/overview.md': regex/index.md + 'guides/vpn/wireguard/overview.md': guides/vpn/wireguard/index.md + 'guides/vpn/openvpn/overview.md': guides/vpn/openvpn/index.md + 'guides/misc/tor/overview.md': guides/misc/tor/index.md + 'guides/github/contributing.md': guides/github/index.md diff --git a/package.json b/package.json index 85e56fed6..fddf89026 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,8 @@ "build": "mkdocs build --clean", "markdownlint": "markdownlint-cli2 \"**/*.md\" \"!**/node_modules/**\"", "linkinator": "npm run build && linkinator site --recurse --silent --skip \"^(?!http://localhost)\"", - "test": "npm run markdownlint && npm run linkinator" + "test": "npm run markdownlint && npm run linkinator", + "serve": "mkdocs serve --dev-addr 0.0.0.0:8000" }, "devDependencies": { "linkinator": "^6.1.2", diff --git a/requirements.txt b/requirements.txt index 879a76104..c341d1f4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ mkdocs==1.6.1 mkdocs-git-revision-date-localized-plugin==1.3.0 -mkdocs-material==9.5.43 -mkdocs-redirects==1.2.1 +mkdocs-material==9.5.44 +mkdocs-redirects==1.2.2 markdown-include==0.8.1