1 /* 2 * Copyright (C) 2017 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 #ifndef LOG_TAG 18 #define LOG_TAG "bpfloader" 19 #endif 20 21 #include <arpa/inet.h> 22 #include <elf.h> 23 #include <error.h> 24 #include <fcntl.h> 25 #include <inttypes.h> 26 #include <linux/bpf.h> 27 #include <linux/unistd.h> 28 #include <net/if.h> 29 #include <stdint.h> 30 #include <stdio.h> 31 #include <stdlib.h> 32 #include <string.h> 33 #include <unistd.h> 34 35 #include <sys/mman.h> 36 #include <sys/socket.h> 37 #include <sys/stat.h> 38 #include <sys/types.h> 39 40 #include <android-base/stringprintf.h> 41 #include <android-base/unique_fd.h> 42 #include <cutils/log.h> 43 44 #include <netdutils/MemBlock.h> 45 #include <netdutils/Misc.h> 46 #include <netdutils/Slice.h> 47 #include "bpf/BpfUtils.h" 48 #include "bpf/bpf_shared.h" 49 50 using android::base::unique_fd; 51 using android::netdutils::MemBlock; 52 using android::netdutils::Slice; 53 54 #define BPF_PROG_PATH "/system/etc/bpf" 55 #define BPF_PROG_SRC BPF_PROG_PATH "/bpf_kern.o" 56 #define MAP_LD_CMD_HEAD 0x18 57 58 #define FAIL(...) \ 59 do { \ 60 ((void)ALOG(LOG_ERROR, LOG_TAG, __VA_ARGS__)); \ 61 exit(-1); \ 62 } while (0) 63 64 // The BPF instruction bytes that we need to replace. x is a placeholder (e.g., COOKIE_TAG_MAP). 65 #define MAP_SEARCH_PATTERN(x) \ 66 { \ 67 0x18, 0x01, 0x00, 0x00, \ 68 (x)[0], (x)[1], (x)[2], (x)[3], \ 69 0x00, 0x00, 0x00, 0x00, \ 70 (x)[4], (x)[5], (x)[6], (x)[7] \ 71 } 72 73 // The bytes we'll replace them with. x is the actual fd number for the map at runtime. 74 // The second byte is changed from 0x01 to 0x11 since 0x11 is the special command used 75 // for bpf map fd loading. The original 0x01 is only a normal load command. 76 #define MAP_REPLACE_PATTERN(x) \ 77 { \ 78 0x18, 0x11, 0x00, 0x00, \ 79 (x)[0], (x)[1], (x)[2], (x)[3], \ 80 0x00, 0x00, 0x00, 0x00, \ 81 (x)[4], (x)[5], (x)[6], (x)[7] \ 82 } 83 84 #define DECLARE_MAP(_mapFd, _mapPath) \ 85 unique_fd _mapFd(android::bpf::mapRetrieve((_mapPath), 0)); \ 86 if (_mapFd < 0) { \ 87 FAIL("Failed to get map from %s", (_mapPath)); \ 88 } 89 90 #define MAP_CMD_SIZE 16 91 #define LOG_BUF_SIZE 65536 92 93 namespace android { 94 namespace bpf { 95 96 struct ReplacePattern { 97 std::array<uint8_t, MAP_CMD_SIZE> search; 98 std::array<uint8_t, MAP_CMD_SIZE> replace; 99 100 ReplacePattern(uint64_t dummyFd, int realFd) { 101 // Ensure that the fd numbers are big-endian. 102 uint8_t beDummyFd[sizeof(uint64_t)]; 103 uint8_t beRealFd[sizeof(uint64_t)]; 104 for (size_t i = 0; i < sizeof(uint64_t); i++) { 105 beDummyFd[i] = (dummyFd >> (i * 8)) & 0xFF; 106 beRealFd[i] = (realFd >> (i * 8)) & 0xFF; 107 } 108 search = MAP_SEARCH_PATTERN(beDummyFd); 109 replace = MAP_REPLACE_PATTERN(beRealFd); 110 } 111 }; 112 113 MemBlock cgroupIngressProg; 114 MemBlock cgroupEgressProg; 115 MemBlock xtIngressProg; 116 MemBlock xtEgressProg; 117 118 MemBlock getProgFromMem(Slice buffer, Elf64_Shdr* section) { 119 uint64_t progSize = (uint64_t)section->sh_size; 120 Slice progSection = take(drop(buffer, section->sh_offset), progSize); 121 if (progSection.size() < progSize) FAIL("programSection out of bound\n"); 122 123 MemBlock progCopy(progSection); 124 if (progCopy.get().size() != progSize) { 125 FAIL("program cannot be extracted"); 126 } 127 return progCopy; 128 } 129 130 void parseProgramsFromFile(const char* path) { 131 int fd = open(path, O_RDONLY); 132 if (fd == -1) { 133 FAIL("Failed to open %s program: %s", path, strerror(errno)); 134 } 135 136 struct stat stat; 137 if (fstat(fd, &stat)) FAIL("Fail to get file size"); 138 139 off_t fileLen = stat.st_size; 140 char* baseAddr = (char*)mmap(NULL, fileLen, PROT_READ, MAP_PRIVATE, fd, 0); 141 if (baseAddr == MAP_FAILED) FAIL("Failed to map the program into memory"); 142 143 if ((uint32_t)fileLen < sizeof(Elf64_Ehdr)) FAIL("file size too small for Elf64_Ehdr"); 144 145 Slice buffer(baseAddr, fileLen); 146 147 Slice elfHeader = take(buffer, sizeof(Elf64_Ehdr)); 148 149 if (elfHeader.size() < sizeof(Elf64_Ehdr)) FAIL("bpf buffer does not have complete elf header"); 150 151 Elf64_Ehdr* elf = (Elf64_Ehdr*)elfHeader.base(); 152 // Find section names string table. This is the section whose index is e_shstrndx. 153 if (elf->e_shstrndx == SHN_UNDEF) { 154 FAIL("cannot locate namesSection\n"); 155 } 156 size_t totalSectionSize = (elf->e_shnum) * sizeof(Elf64_Shdr); 157 Slice sections = take(drop(buffer, elf->e_shoff), totalSectionSize); 158 if (sections.size() < totalSectionSize) { 159 FAIL("sections corrupted"); 160 } 161 162 Slice namesSection = take(drop(sections, elf->e_shstrndx * sizeof(Elf64_Shdr)), 163 sizeof(Elf64_Shdr)); 164 if (namesSection.size() != sizeof(Elf64_Shdr)) { 165 FAIL("namesSection corrupted"); 166 } 167 size_t strTabOffset = ((Elf64_Shdr*) namesSection.base())->sh_offset; 168 size_t strTabSize = ((Elf64_Shdr*) namesSection.base())->sh_size; 169 170 Slice strTab = take(drop(buffer, strTabOffset), strTabSize); 171 if (strTab.size() < strTabSize) { 172 FAIL("string table out of bound\n"); 173 } 174 175 for (int i = 0; i < elf->e_shnum; i++) { 176 Slice section = take(drop(sections, i * sizeof(Elf64_Shdr)), sizeof(Elf64_Shdr)); 177 if (section.size() < sizeof(Elf64_Shdr)) { 178 FAIL("section %d is out of bound, section size: %zu, header size: %zu, total size: %zu", 179 i, section.size(), sizeof(Elf64_Shdr), sections.size()); 180 } 181 Elf64_Shdr* sectionPtr = (Elf64_Shdr*)section.base(); 182 Slice nameSlice = drop(strTab, sectionPtr->sh_name); 183 if (nameSlice.size() == 0) { 184 FAIL("nameSlice out of bound, i: %d, strTabSize: %zu, sh_name: %u", i, strTabSize, 185 sectionPtr->sh_name); 186 } 187 if (!strcmp((char *)nameSlice.base(), BPF_CGROUP_INGRESS_PROG_NAME)) { 188 cgroupIngressProg = getProgFromMem(buffer, sectionPtr); 189 } else if (!strcmp((char *)nameSlice.base(), BPF_CGROUP_EGRESS_PROG_NAME)) { 190 cgroupEgressProg = getProgFromMem(buffer, sectionPtr); 191 } else if (!strcmp((char *)nameSlice.base(), XT_BPF_INGRESS_PROG_NAME)) { 192 xtIngressProg = getProgFromMem(buffer, sectionPtr); 193 } else if (!strcmp((char *)nameSlice.base(), XT_BPF_EGRESS_PROG_NAME)) { 194 xtEgressProg = getProgFromMem(buffer, sectionPtr); 195 } 196 } 197 } 198 199 int loadProg(Slice prog, bpf_prog_type type, const std::vector<ReplacePattern>& mapPatterns) { 200 if (prog.size() == 0) { 201 FAIL("Couldn't find or parse program type %d", type); 202 } 203 Slice remaining = prog; 204 while (remaining.size() >= MAP_CMD_SIZE) { 205 // Scan the program, examining all possible places that might be the start of a map load 206 // operation (i.e., all bytes of value MAP_LD_CMD_HEAD). 207 // In each of these places, check whether it is the start of one of the patterns we want to 208 // replace, and if so, replace it. 209 Slice mapHead = findFirstMatching(remaining, MAP_LD_CMD_HEAD); 210 if (mapHead.size() < MAP_CMD_SIZE) break; 211 bool replaced = false; 212 for (const auto& pattern : mapPatterns) { 213 if (!memcmp(mapHead.base(), pattern.search.data(), MAP_CMD_SIZE)) { 214 memcpy(mapHead.base(), pattern.replace.data(), MAP_CMD_SIZE); 215 replaced = true; 216 break; 217 } 218 } 219 remaining = drop(mapHead, replaced ? MAP_CMD_SIZE : sizeof(uint8_t)); 220 } 221 char bpf_log_buf[LOG_BUF_SIZE]; 222 Slice bpfLog = Slice(bpf_log_buf, sizeof(bpf_log_buf)); 223 return bpfProgLoad(type, prog, "Apache 2.0", 0, bpfLog); 224 } 225 226 int loadAndAttachProgram(bpf_attach_type type, const char* path, const char* name, 227 std::vector<ReplacePattern> mapPatterns) { 228 229 unique_fd fd; 230 if (type == BPF_CGROUP_INET_INGRESS) { 231 fd.reset(loadProg(cgroupIngressProg, BPF_PROG_TYPE_CGROUP_SKB, mapPatterns)); 232 } else if (type == BPF_CGROUP_INET_EGRESS) { 233 fd.reset(loadProg(cgroupEgressProg, BPF_PROG_TYPE_CGROUP_SKB, mapPatterns)); 234 } else if (!strcmp(name, XT_BPF_INGRESS_PROG_NAME)) { 235 fd.reset(loadProg(xtIngressProg, BPF_PROG_TYPE_SOCKET_FILTER, mapPatterns)); 236 } else if (!strcmp(name, XT_BPF_EGRESS_PROG_NAME)) { 237 fd.reset(loadProg(xtEgressProg, BPF_PROG_TYPE_SOCKET_FILTER, mapPatterns)); 238 } else { 239 FAIL("Unrecognized program type: %s", name); 240 } 241 242 if (fd < 0) { 243 FAIL("load %s failed: %s", name, strerror(errno)); 244 } 245 int ret = 0; 246 if (type == BPF_CGROUP_INET_EGRESS || type == BPF_CGROUP_INET_INGRESS) { 247 unique_fd cg_fd(open(CGROUP_ROOT_PATH, O_DIRECTORY | O_RDONLY | O_CLOEXEC)); 248 if (cg_fd < 0) { 249 FAIL("Failed to open the cgroup directory"); 250 } 251 ret = attachProgram(type, fd, cg_fd); 252 if (ret) { 253 FAIL("%s attach failed: %s", name, strerror(errno)); 254 } 255 } 256 257 ret = mapPin(fd, path); 258 if (ret) { 259 FAIL("Pin %s as file %s failed: %s", name, path, strerror(errno)); 260 } 261 return 0; 262 } 263 264 } // namespace bpf 265 } // namespace android 266 267 using android::bpf::APP_UID_STATS_MAP_PATH; 268 using android::bpf::BPF_EGRESS_PROG_PATH; 269 using android::bpf::BPF_INGRESS_PROG_PATH; 270 using android::bpf::COOKIE_TAG_MAP_PATH; 271 using android::bpf::DOZABLE_UID_MAP_PATH; 272 using android::bpf::IFACE_STATS_MAP_PATH; 273 using android::bpf::POWERSAVE_UID_MAP_PATH; 274 using android::bpf::STANDBY_UID_MAP_PATH; 275 using android::bpf::TAG_STATS_MAP_PATH; 276 using android::bpf::UID_COUNTERSET_MAP_PATH; 277 using android::bpf::UID_STATS_MAP_PATH; 278 using android::bpf::XT_BPF_EGRESS_PROG_PATH; 279 using android::bpf::XT_BPF_INGRESS_PROG_PATH; 280 281 using android::bpf::ReplacePattern; 282 using android::bpf::loadAndAttachProgram; 283 284 static void usage(void) { 285 ALOGE( "Usage: ./bpfloader [-i] [-e]\n" 286 " -i load ingress bpf program\n" 287 " -e load egress bpf program\n" 288 " -p load prerouting xt_bpf program\n" 289 " -m load mangle xt_bpf program\n"); 290 } 291 292 int main(int argc, char** argv) { 293 int ret = 0; 294 DECLARE_MAP(cookieTagMap, COOKIE_TAG_MAP_PATH); 295 DECLARE_MAP(uidCounterSetMap, UID_COUNTERSET_MAP_PATH); 296 DECLARE_MAP(appUidStatsMap, APP_UID_STATS_MAP_PATH); 297 DECLARE_MAP(uidStatsMap, UID_STATS_MAP_PATH); 298 DECLARE_MAP(tagStatsMap, TAG_STATS_MAP_PATH); 299 DECLARE_MAP(ifaceStatsMap, IFACE_STATS_MAP_PATH); 300 DECLARE_MAP(dozableUidMap, DOZABLE_UID_MAP_PATH); 301 DECLARE_MAP(standbyUidMap, STANDBY_UID_MAP_PATH); 302 DECLARE_MAP(powerSaveUidMap, POWERSAVE_UID_MAP_PATH); 303 304 const std::vector<ReplacePattern> mapPatterns = { 305 ReplacePattern(COOKIE_TAG_MAP, cookieTagMap.get()), 306 ReplacePattern(UID_COUNTERSET_MAP, uidCounterSetMap.get()), 307 ReplacePattern(APP_UID_STATS_MAP, appUidStatsMap.get()), 308 ReplacePattern(UID_STATS_MAP, uidStatsMap.get()), 309 ReplacePattern(TAG_STATS_MAP, tagStatsMap.get()), 310 ReplacePattern(IFACE_STATS_MAP, ifaceStatsMap.get()), 311 ReplacePattern(DOZABLE_UID_MAP, dozableUidMap.get()), 312 ReplacePattern(STANDBY_UID_MAP, standbyUidMap.get()), 313 ReplacePattern(POWERSAVE_UID_MAP, powerSaveUidMap.get()), 314 }; 315 316 int opt; 317 bool doIngress = false, doEgress = false, doPrerouting = false, doMangle = false; 318 while ((opt = getopt(argc, argv, "iepm")) != -1) { 319 switch (opt) { 320 case 'i': 321 doIngress = true; 322 break; 323 case 'e': 324 doEgress = true; 325 break; 326 case 'p': 327 doPrerouting = true; 328 break; 329 case 'm': 330 doMangle = true; 331 break; 332 default: 333 usage(); 334 FAIL("unknown argument %c", opt); 335 } 336 } 337 android::bpf::parseProgramsFromFile(BPF_PROG_SRC); 338 339 if (doIngress) { 340 ret = loadAndAttachProgram(BPF_CGROUP_INET_INGRESS, BPF_INGRESS_PROG_PATH, 341 BPF_CGROUP_INGRESS_PROG_NAME, mapPatterns); 342 if (ret) { 343 FAIL("Failed to set up ingress program"); 344 } 345 } 346 if (doEgress) { 347 ret = loadAndAttachProgram(BPF_CGROUP_INET_EGRESS, BPF_EGRESS_PROG_PATH, 348 BPF_CGROUP_EGRESS_PROG_NAME, mapPatterns); 349 if (ret) { 350 FAIL("Failed to set up ingress program"); 351 } 352 } 353 if (doPrerouting) { 354 ret = loadAndAttachProgram(MAX_BPF_ATTACH_TYPE, XT_BPF_INGRESS_PROG_PATH, 355 XT_BPF_INGRESS_PROG_NAME, mapPatterns); 356 if (ret) { 357 FAIL("Failed to set up xt_bpf program"); 358 } 359 } 360 if (doMangle) { 361 ret = loadAndAttachProgram(MAX_BPF_ATTACH_TYPE, XT_BPF_EGRESS_PROG_PATH, 362 XT_BPF_EGRESS_PROG_NAME, mapPatterns); 363 if (ret) { 364 FAIL("Failed to set up xt_bpf program"); 365 } 366 } 367 return ret; 368 } 369
