Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

dnsdist: Add meson support #14724

Draft
wants to merge 17 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 27 additions & 16 deletions .github/workflows/build-and-test-all.yml
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ jobs:
needs: get-runner-container-image
strategy:
matrix:
sanitizers: [ubsan+asan, tsan]
sanitizers: [asan+ubsan, tsan]
features: [least, full]
exclude:
- sanitizers: tsan
Expand Down Expand Up @@ -213,11 +213,14 @@ jobs:
needs: get-runner-container-image
strategy:
matrix:
sanitizers: [ubsan+asan, tsan]
builder: [autotools, meson]
sanitizers: [asan+ubsan, tsan]
features: [least, full]
exclude:
- sanitizers: tsan
features: least
- sanitizers: tsan
builder: meson
container:
image: "${{ needs.get-runner-container-image.outputs.id }}:${{ needs.get-runner-container-image.outputs.tag }}"
env:
Expand Down Expand Up @@ -247,25 +250,31 @@ jobs:
uses: actions/cache@v4
with:
path: ~/.ccache
key: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-ccache-${{ steps.get-stamp.outputs.stamp }}
restore-keys: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-ccache-
key: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-${{ matrix.builder}}-ccache-${{ steps.get-stamp.outputs.stamp }}
restore-keys: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-${{ matrix.builder}}-ccache-
- run: inv ci-install-meson
if: ${{ matrix.builder == 'meson' }}
working-directory: ./pdns/dnsdistdist/
- run: inv ci-install-rust ${{ env.REPO_HOME }}
working-directory: ./pdns/dnsdistdist/
- run: inv ci-build-and-install-quiche ${{ env.REPO_HOME }}
working-directory: ./pdns/dnsdistdist/
- run: inv ci-autoconf
if: ${{ matrix.builder == 'autotools' }}
working-directory: ./pdns/dnsdistdist/
- run: inv ci-dnsdist-configure ${{ matrix.features }}
- run: inv ci-dnsdist-configure ${{ matrix.features }} ${{ matrix.builder }} dnsdist-${{ env.BUILDER_VERSION }}
working-directory: ./pdns/dnsdistdist/
- run: inv ci-make-distdir
if: ${{ matrix.builder == 'autotools' }}
working-directory: ./pdns/dnsdistdist/
- run: inv ci-dnsdist-configure ${{ matrix.features }}
- run: inv ci-dnsdist-make-bear
- run: inv ci-dnsdist-run-unit-tests
- run: inv ci-dnsdist-configure ${{ matrix.features }} ${{ matrix.builder }} dnsdist-${{ env.BUILDER_VERSION }}
if: ${{ matrix.builder == 'autotools' }}
- run: inv ci-dnsdist-make-bear ${{ matrix.builder }}
- run: inv ci-dnsdist-run-unit-tests ${{ matrix.builder }}
- run: inv generate-coverage-info ./testrunner $GITHUB_WORKSPACE
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }}
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' && matrix.builder == 'autotools'}}
- name: Coveralls Parallel dnsdist unit
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' }}
if: ${{ env.COVERAGE == 'yes' && matrix.sanitizers != 'tsan' && matrix.builder == 'autotools' }}
uses: coverallsapp/github-action@v2
with:
flag-name: dnsdist-unit-${{ matrix.features }}-${{ matrix.sanitizers }}
Expand All @@ -274,12 +283,14 @@ jobs:
allow-empty: true
fail-on-error: false
- run: inv ci-make-install
if: ${{ matrix.builder == 'autotools' }}
- run: ccache -s
- run: echo "normalized-branch-name=${{ inputs.branch-name || github.ref_name }}" | tr "/" "-" >> "$GITHUB_ENV"
- name: Store the binaries
if: ${{ matrix.builder == 'autotools' }}
uses: actions/upload-artifact@v4 # this takes 30 seconds, maybe we want to tar
with:
name: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-${{ env.normalized-branch-name }}
name: dnsdist-${{ matrix.features }}-${{ matrix.sanitizers }}-${{ matrix.builder}}-${{ env.normalized-branch-name }}
path: /opt/dnsdist
retention-days: 1

Expand Down Expand Up @@ -521,7 +532,7 @@ jobs:
runs-on: ubuntu-22.04
strategy:
matrix:
sanitizers: [ubsan+asan, tsan]
sanitizers: [asan+ubsan, tsan]
dist_name: [debian]
pdns_repo_version: ['48']
container:
Expand Down Expand Up @@ -567,7 +578,7 @@ jobs:
runs-on: ubuntu-22.04
strategy:
matrix:
sanitizers: [ubsan+asan, tsan]
sanitizers: [asan+ubsan, tsan]
dist_name: [debian]
pdns_repo_version: ['48']
container:
Expand Down Expand Up @@ -616,7 +627,7 @@ jobs:
runs-on: ubuntu-22.04
strategy:
matrix:
sanitizers: [ubsan+asan, tsan]
sanitizers: [asan+ubsan, tsan]
threads: [1, 2, 3, 4, 8]
mthreads: [2048]
shards: [1, 2, 1024]
Expand Down Expand Up @@ -661,7 +672,7 @@ jobs:
runs-on: ubuntu-22.04
strategy:
matrix:
sanitizers: [ubsan+asan, tsan]
sanitizers: [asan+ubsan, tsan]
container:
image: "${{ needs.get-runner-container-image.outputs.id }}:${{ needs.get-runner-container-image.outputs.tag }}"
env:
Expand All @@ -685,7 +696,7 @@ jobs:
- name: Fetch the binaries
uses: actions/download-artifact@v4
with:
name: dnsdist-full-${{ matrix.sanitizers }}-${{ env.normalized-branch-name }}
name: dnsdist-full-${{ matrix.sanitizers }}-autotools-${{ env.normalized-branch-name }}
path: /opt/dnsdist
- run: inv install-clang-runtime
- run: inv install-dnsdist-test-deps $([ "$(. /etc/os-release && echo $VERSION_CODENAME)" = "bullseye" ] && echo "--skipXDP=True")
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/codeql-analysis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -166,12 +166,12 @@ jobs:
if: matrix.product == 'dnsdist'
working-directory: ./pdns/dnsdistdist/
run: |
inv ci-dnsdist-configure full
inv ci-dnsdist-configure full autotools build-dir
- name: Build dnsdist
if: matrix.product == 'dnsdist'
working-directory: ./pdns/dnsdistdist/
run: |
inv ci-dnsdist-make-bear
inv ci-dnsdist-make-bear autotools
- run: ln -s ../../.clang-tidy.full .clang-tidy
if: matrix.product == 'dnsdist'
working-directory: ./pdns/dnsdistdist/
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/coverity.yml
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ jobs:
working-directory: ./pdns/dnsdistdist/
- run: inv ci-build-and-install-quiche ${{ env.REPO_HOME }}
working-directory: ./pdns/dnsdistdist/
- run: inv ci-dnsdist-configure full
- run: inv ci-dnsdist-configure full autotools build-dir
working-directory: ./pdns/dnsdistdist/
- run: inv coverity-make
working-directory: ./pdns/dnsdistdist/
Expand Down
3 changes: 3 additions & 0 deletions ext/lmdb-safe/lmdb-safe.hh
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,9 @@ private:
// * so let's go back
}
#else /* ifndef DNSDIST */
(void)key;
(void)data;
(void)op;
return rc;
#endif
}
Expand Down
24 changes: 18 additions & 6 deletions fuzzing/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,15 @@ By default the targets are linked against a standalone target,
`standalone_fuzz_target_runner.cc`, which does no fuzzing but makes it easy
to check a given test file, or just that the fuzzing targets can be built properly.

This behaviour can be changed via the `LIB_FUZZING_ENGINE` variable, for example
by setting it to `-lFuzzer`, building with clang by setting `CC=clang CXX=clang++`
before running the `configure` and adding `-fsanitize=fuzzer-no-link` to `CFLAGS`
and `CXXFLAGS`. Doing so instructs the compiler to instrument the code for
efficient fuzzing but not to link directly with `-lFuzzer`, which would make
the compilation tests done during the configure phase fail.
This behaviour can be changed via:
- either the `LIB_FUZZING_ENGINE` variable when building with `./configure`
- or the `-Dfuzzer_ldflags` option when building with `meson`

For example, setting `LIB_FUZZING_ENGINE` to `-lFuzzer`, then building with clang
by setting `CC=clang CXX=clang++` before running the `configure`, and adding
`-fsanitize=fuzzer-no-link` to `CFLAGS` and `CXXFLAGS`, instructs the compiler
to instrument the code for efficient fuzzing but not to link directly with
`-lFuzzer`, which would make the compilation tests done during the configure phase fail.

Sanitizers
----------
Expand Down Expand Up @@ -86,6 +89,15 @@ LIB_FUZZING_ENGINE="/usr/lib/clang/11.0.1/lib/linux/libclang_rt.fuzzer-x86_64.a"
make -C pdns -j2 fuzz_targets
```

or, if you are using `meson` to build the authoritative server instead of `./configure`:

```
env CC=clang CXX=clang++ \
CFLAGS=-fsanitize=fuzzer-no-link CXXFLAGS=-fsanitize=fuzzer-no-link \
meson setup .. -Dfuzz-targets=true -Dfuzzer_ldflags=/usr/lib/clang/18/lib/linux/libclang_rt.fuzzer-x86_64.a -Db_sanitize=address,undefined
ninja
```

Now you're ready to run one of the fuzzing targets.
First, copy the starting corpus:

Expand Down
19 changes: 14 additions & 5 deletions meson.build
Original file line number Diff line number Diff line change
Expand Up @@ -415,10 +415,6 @@ conditional_sources = {
'sources': [src_dir / 'lua-record.cc'],
'condition': dep_lua_records.found(),
},
'standalone-fuzz-target-runner': {
'sources': [src_dir / 'standalone_fuzz_target_runner.cc'],
'condition': get_option('fuzz-targets'),
},
}

foreach name, info: conditional_sources
Expand Down Expand Up @@ -944,6 +940,15 @@ if get_option('unit-tests')
endif

if get_option('fuzz-targets')
fuzz_extra_sources = []
fuzzer_ldflags = []
# https://github.com/harfbuzz/harfbuzz/pull/2549/files
if get_option('fuzzer_ldflags') == ''
fuzz_extra_sources += src_dir / 'standalone_fuzz_target_runner.cc'
else
fuzzer_ldflags += get_option('fuzzer_ldflags')
endif

fuzz_targets = [
'moadnsparser',
'packetcache',
Expand All @@ -956,7 +961,9 @@ if get_option('fuzz-targets')
foreach target: fuzz_targets
source_file = src_dir / 'fuzz_' + target.underscorify() + '.cc'
tools += {
'fuzz-target-' + target: { 'main': source_file }
'fuzz-target-' + target: { 'main': source_file,
'link-args': fuzzer_ldflags,
'files-extra': fuzz_extra_sources }
}
endforeach
endif
Expand Down Expand Up @@ -984,6 +991,7 @@ foreach tool, info: tools
export_dynamic = 'export-dynamic' in info ? info['export-dynamic'] : false
files_extra = 'files-extra' in info ? info['files-extra'] : []
deps_extra = 'deps-extra' in info ? info['deps-extra'] : []
link_args = 'link-args' in info ? info['link-args'] : []

set_variable(
var_name,
Expand All @@ -993,6 +1001,7 @@ foreach tool, info: tools
config_h,
files_extra,
export_dynamic: export_dynamic,
link_args: link_args,
dependencies: [
deps,
libpdns_common,
Expand Down
1 change: 1 addition & 0 deletions meson_options.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ option('unit-tests', type: 'boolean', value: false, description: 'Build and run
option('unit-tests-backends', type: 'boolean', value: false, description: 'Build and run backend unit tests')
option('reproducible', type: 'boolean', value: false, description: 'Reproducible builds (for distro maintainers, makes debugging difficult)')
option('fuzz-targets', type: 'boolean', value: false, description: 'Enable fuzzing targets')
option('fuzzer_ldflags', type: 'string', value: '', description: 'Linker flags used for the fuzzing targets (a path to the libFuzzer static library, for example)')
option('verbose-logging', type: 'boolean', value: false, description: 'Enable verbose logging')
option('experimental-pkcs11', type: 'feature', value: 'disabled', description: 'PKCS11 support')
option('experimental-gss-tsig', type: 'feature', value: 'disabled', description: 'GSS-TSIG support')
Expand Down
1 change: 1 addition & 0 deletions pdns/dnsdistdist/NOTICE
1 change: 0 additions & 1 deletion pdns/dnsdistdist/base32.hh

This file was deleted.

4 changes: 4 additions & 0 deletions pdns/dnsdistdist/bpf-filter.cc
Original file line number Diff line number Diff line change
Expand Up @@ -900,6 +900,9 @@ uint64_t BPFFilter::getHits(const ComboAddress& requestor)

BPFFilter::BPFFilter(std::unordered_map<std::string, MapConfiguration>& configs, BPFFilter::MapFormat format, bool external)
{
(void)configs;
(void)format;
(void)external;
}

void BPFFilter::addSocket(int)
Expand Down Expand Up @@ -972,6 +975,7 @@ bool BPFFilter::supportsMatchAction(MatchAction action) const
}
return d_mapFormat == BPFFilter::MapFormat::WithActions;
#endif /* HAVE_EBPF */
(void)action;
return false;
}

Expand Down
4 changes: 2 additions & 2 deletions pdns/dnsdistdist/configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ AC_DEFINE([DNSDIST], [1],
LT_PREREQ([2.2.2])
LT_INIT([disable-static])

CFLAGS="-g -O3 -Wall -Wextra -Wshadow -Wno-unused-parameter -fvisibility=hidden $CFLAGS"
CXXFLAGS="-g -O3 -Wall -Wextra -Wshadow -Wno-unused-parameter -Wmissing-declarations -Wredundant-decls -fvisibility=hidden $CXXFLAGS"
CFLAGS="-g -O3 -Wall -Wextra -Wshadow -fvisibility=hidden $CFLAGS"
CXXFLAGS="-g -O3 -Wall -Wextra -Wshadow -Wmissing-declarations -Wredundant-decls -fvisibility=hidden $CXXFLAGS"

AC_SUBST([pdns_configure_args], ["$ac_configure_args"])
AC_DEFINE_UNQUOTED([DNSDIST_CONFIG_ARGS],
Expand Down
3 changes: 2 additions & 1 deletion pdns/dnsdistdist/dnsdist-carbon.cc
Original file line number Diff line number Diff line change
Expand Up @@ -362,9 +362,10 @@ void Carbon::run(const std::vector<Carbon::Endpoint>& endpoints)
}
#endif /* DISABLE_CARBON */

static time_t s_start = time(nullptr);
static const time_t s_start = time(nullptr);

uint64_t uptimeOfProcess(const std::string& str)
{
(void)str;
return time(nullptr) - s_start;
}
1 change: 1 addition & 0 deletions pdns/dnsdistdist/dnsdist-console.cc
Original file line number Diff line number Diff line change
Expand Up @@ -887,6 +887,7 @@ extern "C"

static char** dnsdist_completion_callback(const char* text, int start, int end)
{
(void)end;
char** matches = nullptr;
if (start == 0) {
// NOLINTNEXTLINE(cppcoreguidelines-pro-type-const-cast): readline
Expand Down
16 changes: 8 additions & 8 deletions pdns/dnsdistdist/dnsdist-downstream-connection.hh
Original file line number Diff line number Diff line change
Expand Up @@ -62,15 +62,15 @@ public:
const auto& it = d_downstreamConnections.find(backendId);
if (it != d_downstreamConnections.end()) {
/* first scan idle connections, more recent first */
auto entry = findUsableConnectionInList(now, freshCutOff, it->second.d_idles, true);
auto entry = findUsableConnectionInList(freshCutOff, it->second.d_idles, true);
if (entry) {
++ds->tcpReusedConnections;
it->second.d_actives.insert(entry);
return entry;
}

/* then scan actives ones, more recent first as well */
entry = findUsableConnectionInList(now, freshCutOff, it->second.d_actives, false);
entry = findUsableConnectionInList(freshCutOff, it->second.d_actives, false);
if (entry) {
++ds->tcpReusedConnections;
return entry;
Expand Down Expand Up @@ -107,8 +107,8 @@ public:
idleCutOff.tv_sec -= s_maxIdleTime;

for (auto dsIt = d_downstreamConnections.begin(); dsIt != d_downstreamConnections.end();) {
cleanUpList(dsIt->second.d_idles, now, freshCutOff, idleCutOff);
cleanUpList(dsIt->second.d_actives, now, freshCutOff, idleCutOff);
cleanUpList(dsIt->second.d_idles, freshCutOff, idleCutOff);
cleanUpList(dsIt->second.d_actives, freshCutOff, idleCutOff);

if (dsIt->second.d_idles.empty() && dsIt->second.d_actives.empty()) {
dsIt = d_downstreamConnections.erase(dsIt);
Expand Down Expand Up @@ -212,7 +212,7 @@ public:
}

protected:
void cleanUpList(list_t& list, const struct timeval& now, const struct timeval& freshCutOff, const struct timeval& idleCutOff)
void cleanUpList(list_t& list, const struct timeval& freshCutOff, const struct timeval& idleCutOff)
{
auto& sidx = list.template get<SequencedTag>();
for (auto connIt = sidx.begin(); connIt != sidx.end();) {
Expand Down Expand Up @@ -248,7 +248,7 @@ protected:
}
}

std::shared_ptr<T> findUsableConnectionInList(const struct timeval& now, const struct timeval& freshCutOff, list_t& list, bool removeIfFound)
std::shared_ptr<T> findUsableConnectionInList(const struct timeval& freshCutOff, list_t& list, bool removeIfFound)
{
auto& sidx = list.template get<SequencedTag>();
for (auto listIt = sidx.begin(); listIt != sidx.end();) {
Expand All @@ -258,7 +258,7 @@ protected:
}

auto& entry = *listIt;
if (isConnectionUsable(entry, now, freshCutOff)) {
if (isConnectionUsable(entry, freshCutOff)) {
entry->setReused();
// make a copy since the iterator will be invalidated after erasing
auto result = entry;
Expand All @@ -280,7 +280,7 @@ protected:
return nullptr;
}

bool isConnectionUsable(const std::shared_ptr<T>& conn, const struct timeval& now, const struct timeval& freshCutOff)
bool isConnectionUsable(const std::shared_ptr<T>& conn, const struct timeval& freshCutOff)
{
if (!conn->canBeReused()) {
return false;
Expand Down
Loading
Loading