Skip to content

Commit 289e97f

Browse files
jbajicKVS85
jbajic
and
KVS85
authored
Support vector-index changes (#605)
* Enable support for OpenMP on x64_64 * Add OpenMP to arm * Update Dockerfile x86_64 * Update CMake command * Add external files for building * Update arm Dockerfile * Remove redundant file * Fix for architecture * Fix path for arm * Add gfortran as library * Fix lapack library path --------- Co-authored-by: Vadim Kondratev <vadim@arangodb.com>
1 parent 9b3e177 commit 289e97f

File tree

6 files changed

+234
-11
lines changed

6 files changed

+234
-11
lines changed

‎containers/buildUbuntuDevel.docker/Dockerfile.arm64‎

Lines changed: 59 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
11
# First rebuild glibc with a special option and provide some .deb files:
2+
ARG COMPILER_VERSION=13
3+
ARG CLANG_VERSION=16
4+
ARG CLANG_FULL_VERSION=16.0.6
25

36
FROM ubuntu:24.04 as glibcdebs
47

@@ -30,16 +33,61 @@ RUN dpkg -i /debs/*.deb
3033

3134
# Finally, build the actual build image based on the above preparation:
3235

36+
# Build OpenMP static library
37+
FROM ubuntu:24.04 as openmp
38+
39+
ARG COMPILER_VERSION
40+
ARG CLANG_VERSION
41+
ARG CLANG_FULL_VERSION
42+
43+
RUN apt-get update --fix-missing && \
44+
apt-get upgrade -y && \
45+
apt-get install -y software-properties-common net-tools wget
46+
47+
WORKDIR /
48+
49+
COPY static/ubuntu.sources ./
50+
51+
RUN cat ubuntu.sources >> /etc/apt/sources.list.d/ubuntu.sources
52+
53+
RUN apt-get update && apt-get install dpkg-dev -y
54+
55+
RUN apt-get source llvm-${CLANG_VERSION}-dev -y && \
56+
apt-get build-dep llvm-${CLANG_VERSION}-dev -y
57+
58+
RUN apt-get update --fix-missing && apt-get install -y \
59+
clang-$CLANG_VERSION clang++-$CLANG_VERSION \
60+
cmake lsb-release wget git ninja-build gnupg
61+
62+
COPY patches/diff_llvm.patch .
63+
64+
WORKDIR /llvm-toolchain-${CLANG_VERSION}-${CLANG_FULL_VERSION}
65+
66+
# Patch omp to enable running when it is a static library
67+
RUN git apply /diff_llvm.patch
68+
69+
WORKDIR build
70+
71+
RUN cmake -G Ninja -DLLVM_ENABLE_PROJECTS="clang;openmp" -DCMAKE_C_COMPILER=clang-${CLANG_VERSION} \
72+
-DCMAKE_CXX_COMPILER=clang++-${CLANG_VERSION} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DLIBOMP_ENABLE_SHARED=OFF \
73+
-DENABLE_OMPT_TOOLS=OFF -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_PREFIX=/usr \
74+
-DENABLE_LIBOMPTARGET=OFF -DLIBOMP_INSTALL_ALIASES=OFF ../llvm && ninja
75+
76+
RUN mkdir -p /opt/omp && cp ./lib/libomp.so.5 /opt/omp/libomp.a
77+
78+
3379
FROM prepare
3480

3581
MAINTAINER Max Neunhoeffer <max@arangodb.com>
3682

3783
ARG ARCH="aarch64"
84+
ARG COMPILER_VERSION
85+
ARG CLANG_VERSION
3886

3987
# We provide gcc 13.2.0 as well as clang 16.0.6 from Ubunutu 24.04
4088

41-
ENV COMPILER_VERSION 13
42-
ENV CLANG_VERSION 16
89+
ENV COMPILER_VERSION=${COMPILER_VERSION}
90+
ENV CLANG_VERSION=${CLANG_VERSION}
4391

4492
RUN apt-get update --fix-missing && \
4593
apt-get install -y software-properties-common net-tools && \
@@ -48,7 +96,14 @@ RUN apt-get update --fix-missing && \
4896

4997
ENV DEBIAN_FRONTEND=noninteractive
5098

51-
RUN apt-get update --fix-missing && apt-get install -y build-essential gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} cmake make bison flex python3 ccache git libjemalloc-dev vim exuberant-ctags gdb fish psmisc sudo debhelper debconf jq wget libdb-dev curl gnupg2 gcovr prometheus bc tcpdump liburing-dev cppcheck clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev libclang-rt-${CLANG_VERSION}-dev nodejs npm && apt-get clean
99+
RUN apt-get update --fix-missing && \
100+
apt-get install -y build-essential gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} \
101+
cmake make bison flex python3 ccache git libjemalloc-dev vim exuberant-ctags gdb fish \
102+
psmisc sudo debhelper debconf jq wget libdb-dev curl gnupg2 gcovr prometheus bc tcpdump \
103+
liburing-dev cppcheck clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} \
104+
libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev \
105+
libclang-rt-${CLANG_VERSION}-dev nodejs npm libomp-16-dev liblapack-dev gfortran \
106+
&& apt-get clean
52107

53108
# yarn for UI build:
54109
RUN npm install -g yarn
@@ -61,6 +116,7 @@ RUN curl -L https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-
61116
#COPY --from=v8 /opt/v8/lib /opt/v8/lib
62117
#COPY --from=v8 /opt/v8/include /opt/v8/include
63118
#COPY --from=v8 /opt/v8/third_party /opt/v8/third_party
119+
COPY --from=openmp /opt/omp /opt/omp
64120

65121
# Build openssl and openldap libraries in specific versions and with static
66122
# libraries, install with prefix /opt:

‎containers/buildUbuntuDevel.docker/Dockerfile.x86-64‎

Lines changed: 63 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
1-
# First rebuild glibc with a special option and provide some .deb files:
1+
# Arguments used through multiple build stages
2+
ARG COMPILER_VERSION=13
3+
ARG CLANG_VERSION=16
4+
ARG CLANG_FULL_VERSION=16.0.6
25

6+
# First rebuild glibc with a special option and provide some .deb files:
37
FROM ubuntu:24.04 as glibcdebs
48

59
COPY ./static /static
@@ -28,18 +32,63 @@ RUN dpkg -i /debs/*.deb
2832
#COPY ./tools/install_v8.sh /tools/install_v8.sh
2933
#RUN /tools/install_v8.sh
3034

35+
# Build OpenMP static library
36+
FROM ubuntu:24.04 as openmp
37+
38+
ARG COMPILER_VERSION
39+
ARG CLANG_VERSION
40+
ARG CLANG_FULL_VERSION
41+
42+
RUN apt-get update --fix-missing && \
43+
apt-get upgrade -y && \
44+
apt-get install -y software-properties-common net-tools wget
45+
46+
WORKDIR /
47+
48+
COPY static/ubuntu.sources ./
49+
50+
RUN cat ubuntu.sources >> /etc/apt/sources.list.d/ubuntu.sources
51+
52+
RUN apt-get update && apt-get install dpkg-dev -y
53+
54+
RUN apt-get source llvm-${CLANG_VERSION}-dev -y && \
55+
apt-get build-dep llvm-${CLANG_VERSION}-dev -y
56+
57+
RUN apt-get update --fix-missing && apt-get install -y \
58+
clang-$CLANG_VERSION clang++-$CLANG_VERSION \
59+
cmake lsb-release wget git ninja-build gnupg
60+
61+
COPY patches/diff_llvm.patch .
62+
63+
WORKDIR /llvm-toolchain-${CLANG_VERSION}-${CLANG_FULL_VERSION}
64+
65+
# Patch omp to enable running when it is a static library
66+
RUN git apply /diff_llvm.patch
67+
68+
WORKDIR build
69+
70+
RUN cmake -G Ninja -DLLVM_ENABLE_PROJECTS="clang;openmp" -DCMAKE_C_COMPILER=clang-${CLANG_VERSION} \
71+
-DCMAKE_CXX_COMPILER=clang++-${CLANG_VERSION} -DCMAKE_BUILD_TYPE=RelWithDebInfo -DLIBOMP_ENABLE_SHARED=OFF \
72+
-DENABLE_OMPT_TOOLS=OFF -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON -DCMAKE_INSTALL_PREFIX=/usr \
73+
-DENABLE_LIBOMPTARGET=OFF -DLIBOMP_INSTALL_ALIASES=OFF ../llvm && ninja
74+
75+
RUN mkdir -p /opt/omp && cp ./lib/libomp.so.5 /opt/omp/libomp.a
76+
77+
3178
# Finally, build the actual build image based on the above preparation:
3279

3380
FROM prepare
3481

3582
MAINTAINER Max Neunhoeffer <max@arangodb.com>
3683

3784
ARG ARCH="x86_64"
85+
ARG COMPILER_VERSION
86+
ARG CLANG_VERSION
3887

3988
# We provide gcc 13.2.0 as well as clang 16.0.6 from Ubunutu 24.04
4089

41-
ENV COMPILER_VERSION 13
42-
ENV CLANG_VERSION 16
90+
ENV COMPILER_VERSION=${COMPILER_VERSION}
91+
ENV CLANG_VERSION=${CLANG_VERSION}
4392

4493
RUN apt-get update --fix-missing && \
4594
apt-get install -y software-properties-common net-tools && \
@@ -48,7 +97,14 @@ RUN apt-get update --fix-missing && \
4897

4998
ENV DEBIAN_FRONTEND=noninteractive
5099

51-
RUN apt-get update --fix-missing && apt-get install -y build-essential gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} cmake make bison flex python3 ccache git libjemalloc-dev vim exuberant-ctags gdb fish psmisc sudo debhelper debconf jq wget libdb-dev curl gnupg2 gcovr prometheus bc tcpdump liburing-dev cppcheck clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev libclang-rt-${CLANG_VERSION}-dev nodejs npm && apt-get clean
100+
RUN apt-get update --fix-missing && \
101+
apt-get install -y build-essential gcc-${COMPILER_VERSION} g++-${COMPILER_VERSION} \
102+
cmake make bison flex python3 ccache git libjemalloc-dev vim exuberant-ctags gdb fish \
103+
psmisc sudo debhelper debconf jq wget libdb-dev curl gnupg2 gcovr prometheus bc tcpdump \
104+
liburing-dev cppcheck clang-${CLANG_VERSION} lldb-${CLANG_VERSION} lld-${CLANG_VERSION} \
105+
libc++-${CLANG_VERSION}-dev libc++abi-${CLANG_VERSION}-dev libclang-common-${CLANG_VERSION}-dev \
106+
libclang-rt-${CLANG_VERSION}-dev nodejs npm libomp-16-dev liblapack-dev gfortran \
107+
&& apt-get clean
52108

53109
# yarn for UI build:
54110
RUN npm install -g yarn
@@ -62,6 +118,9 @@ RUN curl -L https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-
62118
#COPY --from=v8 /opt/v8/include /opt/v8/include
63119
#COPY --from=v8 /opt/v8/third_party /opt/v8/third_party
64120

121+
# Fetch libomp
122+
COPY --from=openmp /opt/omp /opt/omp
123+
65124
# Build openssl and openldap libraries in specific versions and with static
66125
# libraries, install with prefix /opt:
67126

‎containers/buildUbuntuDevel.docker/patches/diff_llvm.patch‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
diff --git a/openmp/runtime/src/ompd-specific.cpp b/openmp/runtime/src/ompd-specific.cpp
2+
index c4018789e..07da27023 100644
3+
--- a/openmp/runtime/src/ompd-specific.cpp
4+
+++ b/openmp/runtime/src/ompd-specific.cpp
5+
@@ -82,6 +82,7 @@ void ompd_init() {
6+
char *libname = NULL;
7+
8+
#if KMP_OS_UNIX
9+
+#if 0
10+
// Find the location of libomp.so thru dladdr and replace the libomp with
11+
// libompd to get the full path of libompd
12+
Dl_info dl_info;
13+
@@ -97,6 +98,7 @@ void ompd_init() {
14+
strncpy(libname, dl_info.dli_fname, lib_path_length);
15+
memcpy(libname + lib_path_length, "/libompd.so\0", 12);
16+
}
17+
+#endif
18+
#endif
19+
20+
const char *ompd_env_var = getenv("OMP_DEBUG");
21+
diff --git a/openmp/runtime/src/ompt-general.cpp b/openmp/runtime/src/ompt-general.cpp
22+
index 55a936575..a71c1aef3 100644
23+
--- a/openmp/runtime/src/ompt-general.cpp
24+
+++ b/openmp/runtime/src/ompt-general.cpp
25+
@@ -164,8 +164,7 @@ ompt_start_tool(unsigned int omp_version, const char *runtime_version) {
26+
// runtime library is linked before the tool. Since glibc 2.2 strong symbols
27+
// don't override weak symbols that have been found before unless the user
28+
// sets the environment variable LD_DYNAMIC_WEAK.
29+
- ompt_start_tool_t next_tool =
30+
- (ompt_start_tool_t)dlsym(RTLD_NEXT, "ompt_start_tool");
31+
+ ompt_start_tool_t next_tool = nullptr;
32+
if (next_tool) {
33+
ret = next_tool(omp_version, runtime_version);
34+
}
35+
@@ -245,14 +244,14 @@ ompt_tool_windows(unsigned int omp_version, const char *runtime_version) {
36+
#endif
37+
38+
static ompt_start_tool_result_t *
39+
-ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
40+
+ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
41+
ompt_start_tool_result_t *ret = NULL;
42+
- ompt_start_tool_t start_tool = NULL;
43+
+ // ompt_start_tool_t start_tool = NULL;
44+
#if KMP_OS_WINDOWS
45+
// Cannot use colon to describe a list of absolute paths on Windows
46+
const char *sep = ";";
47+
#else
48+
- const char *sep = ":";
49+
+ // const char *sep = ":";
50+
#endif
51+
52+
OMPT_VERBOSE_INIT_PRINT("----- START LOGGING OF TOOL REGISTRATION -----\n");
53+
@@ -275,7 +274,7 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
54+
OMPT_VERBOSE_INIT_PRINT("----- END LOGGING OF TOOL REGISTRATION -----\n");
55+
return ret;
56+
}
57+
-
58+
+#if 0
59+
// Try tool-libraries-var ICV
60+
OMPT_VERBOSE_INIT_CONTINUED_PRINT("Failed.\n");
61+
const char *tool_libs = getenv("OMP_TOOL_LIBRARIES");
62+
@@ -348,6 +347,8 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
63+
} else {
64+
OMPT_VERBOSE_INIT_PRINT("No OMP_TOOL_LIBRARIES defined.\n");
65+
}
66+
+#endif
67+
+ return nullptr;
68+
69+
// usable tool found in tool-libraries
70+
if (ret) {
71+
@@ -355,6 +356,7 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
72+
return ret;
73+
}
74+
75+
+#if 0
76+
#if KMP_OS_UNIX
77+
{ // Non-standard: load archer tool if application is built with TSan
78+
const char *fname = "libarcher.so";
79+
@@ -383,6 +385,7 @@ ompt_try_start_tool(unsigned int omp_version, const char *runtime_version) {
80+
}
81+
}
82+
}
83+
+#endif
84+
#endif
85+
OMPT_VERBOSE_INIT_PRINT("No OMP tool loaded.\n");
86+
OMPT_VERBOSE_INIT_PRINT("----- END LOGGING OF TOOL REGISTRATION -----\n");

‎containers/buildUbuntuDevel.docker/static/ubuntu.sources‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
2+
Types: deb-src
3+
URIs: http://archive.ubuntu.com/ubuntu/
4+
Suites: noble noble-updates noble-backports
5+
Components: main universe restricted multiverse
6+
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
7+
8+
## Ubuntu security updates. Aside from URIs and Suites,
9+
## this should mirror your choices in the previous section.
10+
Types: deb-src
11+
URIs: http://security.ubuntu.com/ubuntu/
12+
Suites: noble-security
13+
Components: main universe restricted multiverse
14+
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
15+

‎helper.linux.fish‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ else
2222
end
2323

2424
set -gx UBUNTUBUILDIMAGE_312_NAME arangodb/ubuntubuildarangodb-devel
25-
set -gx UBUNTUBUILDIMAGE_312_TAG 7
25+
set -gx UBUNTUBUILDIMAGE_312_TAG 8
2626
set -gx UBUNTUBUILDIMAGE_312 $UBUNTUBUILDIMAGE_312_NAME:$UBUNTUBUILDIMAGE_312_TAG-$UBUNTUBUILDIMAGE_TAG_ARCH
2727

2828
set -gx UBUNTUBUILDIMAGE_311_NAME $UBUNTUBUILDIMAGE_312_NAME

‎scripts/buildArangoDB312.fish‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,12 @@ if test "$ARCH" = "x86_64" -a (string sub -s 1 -l 1 "$OPENSSLPATH") = "3"
3737
set -xg X86_64_SUFFIX "64"
3838
end
3939

40+
if test "$ARCH" = "x86_64"
41+
set -xg LAPACK_LIB_PATH "/usr/lib/x86_64-linux-gnu/lapack/liblapack.a"
42+
else
43+
set -xg LAPACK_LIB_PATH "/usr/lib/aarch64-linux-gnu/lapack/liblapack.a"
44+
end
45+
4046
set -l pie ""
4147

4248
if test "$STATIC_EXECUTABLES" = ""
@@ -53,15 +59,16 @@ set -g FULLARGS $argv \
5359
-DOPENSSL_ROOT_DIR=/opt \
5460
-DUSE_STRICT_OPENSSL_VERSION=$USE_STRICT_OPENSSL \
5561
-DBUILD_REPO_INFO=$BUILD_REPO_INFO \
56-
-DARANGODB_BUILD_DATE="$ARANGODB_BUILD_DATE"
62+
-DARANGODB_BUILD_DATE="$ARANGODB_BUILD_DATE" \
63+
-DLAPACK_LIBRARIES="$LAPACK_LIB_PATH"
5764

5865
if test "$MAINTAINER" = "On"
5966
set -g FULLARGS $FULLARGS \
60-
-DCMAKE_EXE_LINKER_FLAGS="-Wl,--build-id=sha1 $pie -fno-stack-protector -fuse-ld=lld" \
67+
-DCMAKE_EXE_LINKER_FLAGS="-Wl,--build-id=sha1 $pie -fno-stack-protector -fuse-ld=lld -fopenmp=libomp -L/opt/omp -llapack -lgfortran" \
6168
-DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld"
6269
else
6370
set -g FULLARGS $FULLARGS \
64-
-DCMAKE_EXE_LINKER_FLAGS="-Wl,--build-id=sha1 $pie $inline -fno-stack-protector -fuse-ld=lld " \
71+
-DCMAKE_EXE_LINKER_FLAGS="-Wl,--build-id=sha1 $pie $inline -fno-stack-protector -fuse-ld=lld -fopenmp=libomp -L/opt/omp -llapack -lgfortran" \
6572
-DCMAKE_SHARED_LINKER_FLAGS="-fuse-ld=lld" \
6673
-DUSE_CATCH_TESTS=Off \
6774
-DUSE_GOOGLE_TESTS=Off

0 commit comments

Comments
 (0)